From 0047ccb90250e90409ba0ad5ca9f22aea4f56508 Mon Sep 17 00:00:00 2001 From: Shark Date: Fri, 17 Oct 2025 21:24:41 +0200 Subject: [PATCH] initial test262 scheduler --- Cargo.lock | 4 - crates/yavashark_test262/runner/go.mod | 13 +- crates/yavashark_test262/runner/go.sum | 2 + .../runner/progress/progress.go | 151 +++++++++++++++ .../runner/results/results.go | 124 +++++++++++++ crates/yavashark_test262/runner/run/run.go | 66 +++++-- .../runner/scheduler/scheduler.go | 174 ++++++++++++++++++ 7 files changed, 514 insertions(+), 20 deletions(-) create mode 100644 crates/yavashark_test262/runner/go.sum create mode 100644 crates/yavashark_test262/runner/progress/progress.go create mode 100644 crates/yavashark_test262/runner/scheduler/scheduler.go diff --git a/Cargo.lock b/Cargo.lock index 53abccac1..d2772e1b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6074,10 +6074,6 @@ dependencies = [ "yavashark_lexer", ] -[[package]] -name = "yavashark_realm" -version = "0.1.0" - [[package]] name = "yavashark_string" version = "0.1.0" diff --git a/crates/yavashark_test262/runner/go.mod b/crates/yavashark_test262/runner/go.mod index 45a456b22..4237b9df2 100644 --- a/crates/yavashark_test262/runner/go.mod +++ b/crates/yavashark_test262/runner/go.mod @@ -2,4 +2,15 @@ module yavashark_test262_runner go 1.23.2 -require github.com/BurntSushi/toml v1.5.0 // indirect +require ( + github.com/BurntSushi/toml v1.5.0 + github.com/schollz/progressbar/v3 v3.14.1 +) + +require ( + github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/rivo/uniseg v0.4.7 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/term v0.21.0 // indirect +) diff --git a/crates/yavashark_test262/runner/go.sum b/crates/yavashark_test262/runner/go.sum new file mode 100644 index 000000000..ff7fd092f --- /dev/null +++ b/crates/yavashark_test262/runner/go.sum @@ -0,0 +1,2 @@ +github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= +github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= diff --git a/crates/yavashark_test262/runner/progress/progress.go b/crates/yavashark_test262/runner/progress/progress.go new file mode 100644 index 000000000..84191fe75 --- /dev/null +++ b/crates/yavashark_test262/runner/progress/progress.go @@ -0,0 +1,151 @@ +package progress + +import ( + "fmt" + "sync" + "sync/atomic" + "yavashark_test262_runner/status" +) + +const ( + ColorGreen = "\033[32m" + ColorRed = "\033[31m" + ColorYellow = "\033[33m" + ColorBlue = "\033[34m" + ColorMagenta = "\033[35m" + ColorCyan = "\033[36m" + ColorReset = "\033[0m" + ColorGray = "\033[90m" +) + +type ProgressTracker struct { + mu sync.Mutex + + passed atomic.Uint32 + failed atomic.Uint32 + skipped atomic.Uint32 + timeout atomic.Uint32 + crashed atomic.Uint32 + parseError atomic.Uint32 + parseSuccessError atomic.Uint32 + notImplemented atomic.Uint32 + runnerError atomic.Uint32 + total atomic.Uint32 + lastPrintedProgress atomic.Uint32 + + totalTests uint32 +} + +func NewProgressTracker(totalTests uint32) *ProgressTracker { + return &ProgressTracker{ + totalTests: totalTests, + } +} + +func (pt *ProgressTracker) Add(s status.Status) { + switch s { + case status.PASS: + pt.passed.Add(1) + case status.FAIL: + pt.failed.Add(1) + case status.SKIP: + pt.skipped.Add(1) + case status.TIMEOUT: + pt.timeout.Add(1) + case status.CRASH: + pt.crashed.Add(1) + case status.PARSE_ERROR: + pt.parseError.Add(1) + case status.PARSE_SUCCESS_ERROR: + pt.parseSuccessError.Add(1) + case status.NOT_IMPLEMENTED: + pt.notImplemented.Add(1) + case status.RUNNER_ERROR: + pt.runnerError.Add(1) + } + + current := pt.total.Add(1) + pt.updateProgress(current) +} + +func (pt *ProgressTracker) updateProgress(current uint32) { + lastPrinted := pt.lastPrintedProgress.Load() + + threshold := uint32(100) + if pt.totalTests > 0 { + percentThreshold := pt.totalTests / 50 // 2% + if percentThreshold > threshold { + threshold = percentThreshold + } + } + + if current-lastPrinted >= threshold || current == pt.totalTests { + if pt.lastPrintedProgress.CompareAndSwap(lastPrinted, current) { + pt.printProgressBar(current) + } + } +} + +func (pt *ProgressTracker) printProgressBar(current uint32) { + passed := pt.passed.Load() + failed := pt.failed.Load() + skipped := pt.skipped.Load() + timeout := pt.timeout.Load() + crashed := pt.crashed.Load() + + barWidth := 50 + passedWidth := int(float64(passed) / float64(current) * float64(barWidth)) + failedWidth := int(float64(failed) / float64(current) * float64(barWidth)) + skippedWidth := int(float64(skipped) / float64(current) * float64(barWidth)) + timeoutWidth := int(float64(timeout) / float64(current) * float64(barWidth)) + crashedWidth := int(float64(crashed) / float64(current) * float64(barWidth)) + otherWidth := barWidth - passedWidth - failedWidth - skippedWidth - timeoutWidth - crashedWidth + + bar := "" + if passedWidth > 0 { + bar += ColorGreen + repeatChar("█", passedWidth) + ColorReset + } + if failedWidth > 0 { + bar += ColorRed + repeatChar("█", failedWidth) + ColorReset + } + if timeoutWidth > 0 { + bar += ColorYellow + repeatChar("█", timeoutWidth) + ColorReset + } + if crashedWidth > 0 { + bar += ColorMagenta + repeatChar("█", crashedWidth) + ColorReset + } + if skippedWidth > 0 { + bar += ColorCyan + repeatChar("█", skippedWidth) + ColorReset + } + if otherWidth > 0 { + bar += ColorGray + repeatChar("░", otherWidth) + ColorReset + } + + percentage := float64(current) / float64(pt.totalTests) * 100 + + fmt.Printf("\r[%s] %3d%% (%d/%d) | %sP:%d%s %sF:%d%s %sT:%d%s %sC:%d%s %sS:%d%s", + bar, + int(percentage), + current, + pt.totalTests, + ColorGreen, passed, ColorReset, + ColorRed, failed, ColorReset, + ColorYellow, timeout, ColorReset, + ColorMagenta, crashed, ColorReset, + ColorCyan, skipped, ColorReset, + ) +} + +func (pt *ProgressTracker) GetStats() (passed, failed, skipped, timeout, crashed, parseError, parseSuccessError, notImplemented, runnerError, total uint32) { + return pt.passed.Load(), pt.failed.Load(), pt.skipped.Load(), pt.timeout.Load(), pt.crashed.Load(), + pt.parseError.Load(), pt.parseSuccessError.Load(), pt.notImplemented.Load(), pt.runnerError.Load(), + pt.total.Load() +} + +func repeatChar(char string, count int) string { + result := "" + for i := 0; i < count; i++ { + result += char + } + return result +} diff --git a/crates/yavashark_test262/runner/results/results.go b/crates/yavashark_test262/runner/results/results.go index ca18fc23e..6a869e73e 100644 --- a/crates/yavashark_test262/runner/results/results.go +++ b/crates/yavashark_test262/runner/results/results.go @@ -8,6 +8,18 @@ import ( "yavashark_test262_runner/status" ) +const ( + ColorGreen = "\033[32m" + ColorRed = "\033[31m" + ColorYellow = "\033[33m" + ColorBlue = "\033[34m" + ColorMagenta = "\033[35m" + ColorCyan = "\033[36m" + ColorReset = "\033[0m" + ColorBold = "\033[1m" + ColorDim = "\033[2m" +) + type TestResults struct { TestResults []Result Passed uint32 @@ -212,7 +224,119 @@ func (tr *TestResults) Compare(other *TestResults) { printDiff("Timeout", tr.Timeout, other.Timeout, tr.Total) printDiff("Parse Error", tr.ParseError, other.ParseError, tr.Total) printDiff("Total", tr.Total, other.Total, tr.Total) +} + +func (tr *TestResults) PrintResultsWithDiff(other *TestResults) { + fmt.Printf("\n%s=== Test Results Summary ===%s\n\n", ColorBold, ColorReset) + + fmt.Printf("%sCurrent Run:%s\n", ColorBold, ColorReset) + tr.printResultsLine("Passed", tr.Passed, tr.Total, other.Passed) + tr.printResultsLine("Failed", tr.Failed, tr.Total, other.Failed) + tr.printResultsLine("Timeout", tr.Timeout, tr.Total, other.Timeout) + tr.printResultsLine("Crashed", tr.Crashed, tr.Total, other.Crashed) + tr.printResultsLine("Skipped", tr.Skipped, tr.Total, other.Skipped) + tr.printResultsLine("Not Implemented", tr.NotImplemented, tr.Total, other.NotImplemented) + tr.printResultsLine("Runner Error", tr.RunnerError, tr.Total, other.RunnerError) + tr.printResultsLine("Parse Error", tr.ParseError, tr.Total, other.ParseError) + tr.printResultsLine("Parse Success Error", tr.ParseSuccessError, tr.Total, other.ParseSuccessError) + + fmt.Printf("\n%sTotal: %d%s\n", ColorBold, tr.Total, ColorReset) + + fmt.Printf("\n%s=== Net Changes ===%s\n", ColorBold, ColorReset) + netPassed := int32(tr.Passed) - int32(other.Passed) + netFailed := int32(tr.Failed) - int32(other.Failed) + netTimeout := int32(tr.Timeout) - int32(other.Timeout) + netCrashed := int32(tr.Crashed) - int32(other.Crashed) + + if netPassed != 0 { + if netPassed > 0 { + fmt.Printf("%s✓ Passed: +%d%s (gained)\n", ColorGreen, netPassed, ColorReset) + } else { + fmt.Printf("%s✗ Passed: %d%s (lost)\n", ColorRed, netPassed, ColorReset) + } + } + + if netFailed != 0 { + if netFailed > 0 { + fmt.Printf("%s✗ Failed: +%d%s (gained)\n", ColorRed, netFailed, ColorReset) + } else { + fmt.Printf("%s✓ Failed: %d%s (improved)\n", ColorGreen, netFailed, ColorReset) + } + } + + if netTimeout != 0 { + if netTimeout > 0 { + fmt.Printf("%s⏱ Timeout: +%d%s (gained)\n", ColorYellow, netTimeout, ColorReset) + } else { + fmt.Printf("%s✓ Timeout: %d%s (improved)\n", ColorGreen, netTimeout, ColorReset) + } + } + + if netCrashed != 0 { + if netCrashed > 0 { + fmt.Printf("%s💥 Crashed: +%d%s (gained)\n", ColorMagenta, netCrashed, ColorReset) + } else { + fmt.Printf("%s✓ Crashed: %d%s (improved)\n", ColorGreen, netCrashed, ColorReset) + } + } + + // Overall summary + fmt.Printf("\n%s=== Overall Summary ===%s\n", ColorBold, ColorReset) + totalChanges := abs(netPassed) + abs(netFailed) + abs(netTimeout) + abs(netCrashed) + if totalChanges > 0 { + fmt.Printf("Total test status changes: %d\n", totalChanges) + + passedGained := 0 + if netPassed > 0 { + passedGained = int(netPassed) + } + failedLost := 0 + if netFailed < 0 { + failedLost = int(-netFailed) + } + improvements := passedGained + failedLost + if improvements > 0 { + fmt.Printf("%s↑ Improvements: %d%s\n", ColorGreen, improvements, ColorReset) + } + + failedGained := 0 + if netFailed > 0 { + failedGained = int(netFailed) + } + passedLost := 0 + if netPassed < 0 { + passedLost = int(-netPassed) + } + regressions := failedGained + passedLost + + if regressions > 0 { + fmt.Printf("%s↓ Regressions: %d%s\n", ColorRed, regressions, ColorReset) + } + } else { + fmt.Printf("%sNo changes from previous run%s\n", ColorDim, ColorReset) + } +} + +func (tr *TestResults) printResultsLine(name string, current, total, previous uint32) { + percentage := float64(current) / float64(total) * 100 + diff := int32(current) - int32(previous) + + var diffStr string + if diff > 0 { + diffStr = fmt.Sprintf(" %s(+%d)%s", ColorGreen, diff, ColorReset) + } else if diff < 0 { + diffStr = fmt.Sprintf(" %s(%d)%s", ColorRed, diff, ColorReset) + } + + fmt.Printf(" %s: %d (%.2f%%)%s\n", name, current, percentage, diffStr) +} + +func abs(n int32) int32 { + if n < 0 { + return -n + } + return n } func printDiff(name string, n1 uint32, n2 uint32, total uint32) { diff --git a/crates/yavashark_test262/runner/run/run.go b/crates/yavashark_test262/runner/run/run.go index 664c047ee..c853575ed 100644 --- a/crates/yavashark_test262/runner/run/run.go +++ b/crates/yavashark_test262/runner/run/run.go @@ -1,13 +1,16 @@ package run import ( + "fmt" "log" "os" "path/filepath" "strings" "sync" "time" + "yavashark_test262_runner/progress" "yavashark_test262_runner/results" + "yavashark_test262_runner/scheduler" "yavashark_test262_runner/status" "yavashark_test262_runner/worker" ) @@ -26,24 +29,28 @@ func TestsInDir(testRoot string, workers int, skips bool, timings bool) *results wg.Add(workers) + num := countTests(testRoot) + progressTracker := progress.NewProgressTracker(num) + for i := range workers { go worker.Worker(i, jobs, resultsChan, wg, timings) } - num := countTests(testRoot) - testResults := results.New(num) + // Goroutine to process results and update progress go func() { for res := range resultsChan { testResults.Add(res) + progressTracker.Add(res.Status) } }() - now := time.Now() + var testPaths []string + var skippedPaths []string + _ = filepath.Walk(testRoot, func(path string, info os.FileInfo, err error) error { if err != nil { - //log.Printf("Failed to get file info for %s: %v", path, err) return nil } @@ -61,30 +68,59 @@ func TestsInDir(testRoot string, workers int, skips bool, timings bool) *results return nil } + shouldSkip := false if skips { for _, skip := range SKIP { if strings.HasPrefix(p, skip) { - resultsChan <- results.Result{ - Status: status.SKIP, - Msg: "skip", - Path: path, - MemoryKB: 0, - Duration: 0, - } - - return nil + skippedPaths = append(skippedPaths, path) + shouldSkip = true + break } } } - jobs <- path + if !shouldSkip { + testPaths = append(testPaths, path) + } return nil }) - close(jobs) + timingData := scheduler.LoadTestTimings("results.json") + + scheduler.EnrichTimingsWithFallback(timingData, testPaths) + + min, max, avg, fastCount, mediumCount, slowCount, riskCount := scheduler.GetStatistics(timingData) + log.Printf("Timing statistics - Min: %v, Max: %v, Avg: %v", min, max, avg) + log.Printf("Test distribution - Fast: %d, Medium: %d, Slow: %d, Risky: %d", + fastCount, mediumCount, slowCount, riskCount) + + scheduledJobs := scheduler.ScheduleTests(testPaths, timingData) + + now := time.Now() + + go func() { + for _, job := range scheduledJobs { + jobs <- job.Path + } + + for _, path := range skippedPaths { + resultsChan <- results.Result{ + Status: status.SKIP, + Msg: "skip", + Path: path, + MemoryKB: 0, + Duration: 0, + } + } + + close(jobs) + }() wg.Wait() + + fmt.Printf("\n") + log.Printf("Finished running %d tests in %s", num, time.Since(now).String()) close(resultsChan) diff --git a/crates/yavashark_test262/runner/scheduler/scheduler.go b/crates/yavashark_test262/runner/scheduler/scheduler.go new file mode 100644 index 000000000..acc255c86 --- /dev/null +++ b/crates/yavashark_test262/runner/scheduler/scheduler.go @@ -0,0 +1,174 @@ +package scheduler + +import ( + "encoding/json" + "log" + "os" + "sort" + "time" +) + +type TestJob struct { + Path string + EstimatedTime time.Duration + Priority int +} + +type StoredResult struct { + Status string `json:"status"` + Msg string `json:"msg"` + Path string `json:"path"` + MemoryKB uint64 `json:"memory_kb"` + Duration time.Duration `json:"duration"` +} + +const ( + // Priority levels + PRIORITY_FAST = 0 // Tests < 100ms based on history + PRIORITY_MEDIUM = 1 // Tests 100ms - 1s + PRIORITY_SLOW = 2 // Tests 1s - 5s + PRIORITY_SLOW_RISK = 3 // Tests > 5s or timeout/crash +) + +// ScheduleTests sorts tests intelligently using historical timing data +func ScheduleTests(testPaths []string, timings map[string]time.Duration) []TestJob { + jobs := make([]TestJob, len(testPaths)) + + for i, path := range testPaths { + estimatedTime := timings[path] + priority := calculatePriorityFromTiming(estimatedTime) + + jobs[i] = TestJob{ + Path: path, + EstimatedTime: estimatedTime, + Priority: priority, + } + } + + sort.Slice(jobs, func(i, j int) bool { + if jobs[i].Priority != jobs[j].Priority { + return jobs[i].Priority < jobs[j].Priority + } + if jobs[i].EstimatedTime != jobs[j].EstimatedTime { + return jobs[i].EstimatedTime < jobs[j].EstimatedTime + } + return jobs[i].Path < jobs[j].Path + }) + + return jobs +} + +func calculatePriorityFromTiming(duration time.Duration) int { + if duration == 0 { + return PRIORITY_MEDIUM + } + + if duration > 5*time.Second { + return PRIORITY_SLOW_RISK + } else if duration > 1*time.Second { + return PRIORITY_SLOW + } else if duration > 100*time.Millisecond { + return PRIORITY_MEDIUM + } + return PRIORITY_FAST +} + +func LoadTestTimings(resultsPath string) map[string]time.Duration { + timings := make(map[string]time.Duration) + + contents, err := os.ReadFile(resultsPath) + if err != nil { + if os.IsNotExist(err) { + log.Printf("No previous results found at %s, using default priorities", resultsPath) + return timings + } + log.Printf("Failed to read results file %s: %v", resultsPath, err) + return timings + } + + var results []StoredResult + err = json.Unmarshal(contents, &results) + if err != nil { + log.Printf("Failed to parse results JSON: %v", err) + return timings + } + + for _, result := range results { + path := result.Path + + if result.Status == "TIMEOUT" { + timings[path] = 30 * time.Second + } else if result.Status == "CRASH" { + timings[path] = 20 * time.Second + } else if result.Duration > 0 { + timings[path] = result.Duration + } else { + timings[path] = 500 * time.Millisecond + } + } + + log.Printf("Loaded timing data for %d tests from %s", len(timings), resultsPath) + return timings +} + +func EstimateTimingFromFileSize(path string) time.Duration { + fileInfo, err := os.Stat(path) + if err != nil { + return 500 * time.Millisecond + } + + sizeKB := fileInfo.Size() / 1024 + + if sizeKB > 100 { + return 10 * time.Second + } else if sizeKB > 50 { + return 5 * time.Second + } else if sizeKB > 20 { + return 1 * time.Second + } else if sizeKB > 5 { + return 200 * time.Millisecond + } + return 50 * time.Millisecond +} + +func EnrichTimingsWithFallback(timings map[string]time.Duration, testPaths []string) { + for _, path := range testPaths { + if _, exists := timings[path]; !exists { + timings[path] = EstimateTimingFromFileSize(path) + } + } +} + +func GetStatistics(timings map[string]time.Duration) (min, max, avg time.Duration, fastCount, mediumCount, slowCount, riskCount int) { + if len(timings) == 0 { + return + } + + var total time.Duration + + for _, duration := range timings { + total += duration + + priority := calculatePriorityFromTiming(duration) + switch priority { + case PRIORITY_FAST: + fastCount++ + case PRIORITY_MEDIUM: + mediumCount++ + case PRIORITY_SLOW: + slowCount++ + case PRIORITY_SLOW_RISK: + riskCount++ + } + + if min == 0 || duration < min { + min = duration + } + if duration > max { + max = duration + } + } + + avg = total / time.Duration(len(timings)) + return +}