builder: simplify running of jobs

Instead of keeping a slice of jobs to run, let the runJobs function
determine which jobs should be run by investigating all dependencies.
This has two benefits:

  - The code is somewhat cleaner, as no 'jobs' slice needs to be
    maintained while constructing the dependency graph.
  - Eventually, some jobs might not be required by any dependency.
    While it's possible to avoid adding them to the slice, the simpler
    solution is to build a new slice from the dependencies which will
    only include required dependencies by design.

Author: aykevl

builder/build.go

@@ -124,11 +124,6 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 		return err
 	}
 
-	// The slice of jobs that orchestrates most of the build.
-	// This is somewhat like an in-memory Makefile with each job being a
-	// Makefile target.
-	var jobs []*compileJob
-
 	// Create the *ssa.Program. This does not yet build the entire SSA of the
 	// program so it's pretty fast and doesn't need to be parallelized.
 	program := lprogram.LoadSSA()
@@ -311,7 +306,6 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 				return os.Rename(f.Name(), bitcodePath)
 			},
 		}
-		jobs = append(jobs, job)
 		packageJobs = append(packageJobs, job)
 	}
 
@@ -402,14 +396,13 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 			return nil
 		},
 	}
-	jobs = append(jobs, programJob)
 
 	// Check whether we only need to create an object file.
 	// If so, we don't need to link anything and will be finished quickly.
 	outext := filepath.Ext(outpath)
 	if outext == ".o" || outext == ".bc" || outext == ".ll" {
 		// Run jobs to produce the LLVM module.
-		err := runJobs(jobs)
+		err := runJobs(programJob)
 		if err != nil {
 			return err
 		}
@@ -450,7 +443,6 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 			return ioutil.WriteFile(objfile, llvmBuf.Bytes(), 0666)
 		},
 	}
-	jobs = append(jobs, outputObjectFileJob)
 
 	// Prepare link command.
 	linkerDependencies := []*compileJob{outputObjectFileJob}
@@ -465,8 +457,6 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 		if err != nil {
 			return err
 		}
-		jobs = append(jobs, job.dependencies...)
-		jobs = append(jobs, job)
 		linkerDependencies = append(linkerDependencies, job)
 	}
 
@@ -484,7 +474,6 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 				return err
 			},
 		}
-		jobs = append(jobs, job)
 		linkerDependencies = append(linkerDependencies, job)
 	}
 
@@ -503,7 +492,6 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 					return err
 				},
 			}
-			jobs = append(jobs, job)
 			linkerDependencies = append(linkerDependencies, job)
 		}
 	}
@@ -521,17 +509,13 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 		if err != nil {
 			return err
 		}
-		// The library needs to be compiled (cache miss).
-		jobs = append(jobs, job.dependencies...)
-		jobs = append(jobs, job)
 		linkerDependencies = append(linkerDependencies, job)
 	case "wasi-libc":
 		path := filepath.Join(root, "lib/wasi-libc/sysroot/lib/wasm32-wasi/libc.a")
 		if _, err := os.Stat(path); os.IsNotExist(err) {
 			return errors.New("could not find wasi-libc, perhaps you need to run `make wasi-libc`?")
 		}
 		job := dummyCompileJob(path)
-		jobs = append(jobs, job)
 		linkerDependencies = append(linkerDependencies, job)
 	case "":
 		// no library specified, so nothing to do
@@ -574,7 +558,7 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 
 	// Create a linker job, which links all object files together and does some
 	// extra stuff that can only be done after linking.
-	jobs = append(jobs, &compileJob{
+	linkJob := &compileJob{
 		description:  "link",
 		dependencies: linkerDependencies,
 		run: func(job *compileJob) error {
@@ -647,12 +631,12 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 
 			return nil
 		},
-	})
+	}
 
 	// Run all jobs to compile and link the program.
 	// Do this now (instead of after elf-to-hex and similar conversions) as it
 	// is simpler and cannot be parallelized.
-	err = runJobs(jobs)
+	err = runJobs(linkJob)
 	if err != nil {
 		return err
 	}

builder/jobs.go

@@ -65,12 +65,29 @@ func (job *compileJob) readyToRun() bool {
 	return true
 }
 
-// runJobs runs all the jobs indicated in the jobs slice and returns the error
-// of the first job that fails to run.
-// It runs all jobs in the order of the slice, as long as all dependencies have
-// already run. Therefore, if some jobs are preferred to run before others, they
-// should be ordered as such in this slice.
-func runJobs(jobs []*compileJob) error {
+// runJobs runs the indicated job and all its dependencies. For every job, all
+// the dependencies are run first. It returns the error of the first job that
+// fails.
+// It runs all jobs in the order of the dependencies slice, depth-first.
+// Therefore, if some jobs are preferred to run before others, they should be
+// ordered as such in the job dependencies.
+func runJobs(job *compileJob) error {
+	// Create a slice of jobs to run, where all dependencies are run in order.
+	jobs := []*compileJob{}
+	addedJobs := map[*compileJob]struct{}{}
+	var addJobs func(*compileJob)
+	addJobs = func(job *compileJob) {
+		if _, ok := addedJobs[job]; ok {
+			return
+		}
+		for _, dep := range job.dependencies {
+			addJobs(dep)
+		}
+		jobs = append(jobs, job)
+		addedJobs[job] = struct{}{}
+	}
+	addJobs(job)
+
 	// Create channels to communicate with the workers.
 	doneChan := make(chan *compileJob)
 	workerChan := make(chan *compileJob)

builder/library.go

@@ -46,15 +46,14 @@ func (l *Library) Load(target, tmpdir string) (path string, err error) {
 	if err != nil {
 		return "", err
 	}
-	jobs := append([]*compileJob{job}, job.dependencies...)
-	err = runJobs(jobs)
+	err = runJobs(job)
 	return job.result, err
 }
 
 // load returns a compile job to build this library file for the given target
 // and CPU. It may return a dummy compileJob if the library build is already
-// cached. The path is stored as job.result but is only valid if the job and
-// job.dependencies have been run.
+// cached. The path is stored as job.result but is only valid after the job has
+// been run.
 // The provided tmpdir will be used to store intermediary files and possibly the
 // output archive file, it is expected to be removed after use.
 func (l *Library) load(target, cpu, tmpdir string) (job *compileJob, err error) {