[Compiler Inductor Query Improvement] Move compiler query to use QueryBuilder (#7464)

# Overview
onvert compiler query to queryBuilder for dynamically build the query

# Details
The fetch performance slows down due to 
the where statement to filter benchmark_extra.output 
Investigated, the noisy output end with ".csv" only affect rocm and cuda
a100. Add the addtional filter where statement only for situation for
rocm and A100

# Tested
tested with notification, the api does not break the regression summary
report, since it also accept single word fields

Author: yangw-dev

torchci/components/benchmark_v3/configs/teams/compilers/config.ts

@@ -140,6 +140,21 @@ const RENDER_MAPPING_BOOK = {
   },
 };
 
+export function toQueryArch(device: string, arch: string) {
+  if (arch === undefined) return [];
+  if (!device) return [];
+  switch (device) {
+    case "rocm":
+      if (arch === "mi300x" || arch == "") return ["mi300x", "mi325x"];
+      return [arch];
+    default:
+      if (arch === "") {
+        return [];
+      }
+      return [arch];
+  }
+}
+
 export const compilerQueryParameterConverter: QueryParameterConverter = (
   inputs: QueryParameterConverterInputs
 ) => {
@@ -154,16 +169,21 @@ export const compilerQueryParameterConverter: QueryParameterConverter = (
   }
 
   let models = getModels(f.model);
+
+  const device = DISPLAY_NAMES_TO_DEVICE_NAMES[f.deviceName];
+  const arch = DISPLAY_NAMES_TO_ARCH_NAMES[f.deviceName];
+  const arches = toQueryArch(device, arch);
+
   const params = {
     commits: i.commits ?? [],
     branches: i.branches ?? [],
     workflows: workflows,
     compilers: compilerList,
-    arch: DISPLAY_NAMES_TO_ARCH_NAMES[f.deviceName],
-    device: DISPLAY_NAMES_TO_DEVICE_NAMES[f.deviceName],
-    dtype: f.dtype === "none" ? "" : f.dtype,
+    arches: arches,
+    devices: [device],
+    dtypes: f.dtype === "none" ? [] : [f.dtype],
     granularity: "hour",
-    mode: f.mode,
+    modes: [f.mode],
     models: models,
     startTime: dayjs.utc(i.timeRange.start).format("YYYY-MM-DDTHH:mm:ss"),
     stopTime: dayjs.utc(i.timeRange.end).format("YYYY-MM-DDTHH:mm:ss"),

torchci/lib/benchmark/api_helper/backend/common/type.ts

@@ -37,10 +37,10 @@ export const defaultCompilerGetBenchmarkDataInputs: any = {
 
 export const defaultListCommitsInputs: any = {
   branches: [],
-  device: "",
+  devices: [],
   arch: [],
-  dtype: "",
-  mode: "",
+  dtypes: [],
+  modes: [],
   startTime: "",
   stopTime: "",
   suites: [],

torchci/lib/benchmark/api_helper/backend/common/utils.ts

@@ -1,7 +1,7 @@
 // Utility to extract params from either GET or POST
 import dayjs from "dayjs";
-import { queryClickhouseSaved } from "lib/clickhouse";
 import { NextApiRequest } from "next";
+import { BenchmarkCompilerListCommitQueryBuilder } from "../dataFetchers/queryBuilderUtils/compilerQueryBuilder";
 import { CommitResult } from "./type";
 
 /**
@@ -380,11 +380,19 @@ function subsampleCommitsByDate(data: any[], maxCount: number | undefined) {
   };
 }
 
+async function listCommitsFromDb(queryParams: any) {
+  // fetch metadata from db
+  const fetcher = new BenchmarkCompilerListCommitQueryBuilder();
+  const data = await fetcher.applyQuery(queryParams);
+  const result = fetcher.postProcess(data);
+  return result;
+}
+
 export async function getCommitsWithSampling(
   tableName: string,
   queryParams: any
 ): Promise<CommitResult> {
-  const commit_results = await queryClickhouseSaved(tableName, queryParams);
+  const commit_results = await listCommitsFromDb(queryParams);
   let maxCount = undefined;
   // if subsampling is specified, use it
   if (queryParams.sampling) {

torchci/lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data.ts

@@ -1,20 +1,14 @@
-import { queryClickhouseSaved } from "lib/clickhouse";
 import {
   CommitResult,
   CompilerQueryType,
-  defaultCompilerGetBenchmarkDataInputs,
   defaultCompilerGetTimeSeriesInputs,
-  defaultListCommitsInputs,
 } from "../common/type";
 import {
   emptyTimeSeriesResponse,
   getCommitsWithSampling,
 } from "../common/utils";
-import {
-  extractBackendSqlStyle,
-  toApiArch,
-  toQueryArch,
-} from "./helpers/common";
+import { BenchmarkCompilerBenchmarkDataQueryBuilder } from "../dataFetchers/queryBuilderUtils/compilerQueryBuilder";
+import { extractBackendSqlStyle, toApiArch } from "./helpers/common";
 import { toGeneralCompilerData } from "./helpers/general";
 import { toPrecomputeCompilerData } from "./helpers/precompute";
 
@@ -23,26 +17,6 @@ const COMPILER_BENCHMARK_TABLE_NAME = "compilers_benchmark_api_query";
 const COMPILER_BENCHMARK_COMMITS_TABLE_NAME =
   "compilers_benchmark_api_commit_query";
 
-// TODO(ELAINEWY): add GET BENCHMARK DATA API
-/**
- * backend method to get single compiler benchmark data
- * must provide workflow and branch in inputParams
- */
-export async function getSingleCompilerBenchmarkData(
-  request_name: string,
-  inputParams: any,
-  formats: string[] = ["raw"]
-) {
-  const queryParams = await getSingleCompilerBenchmarkDataQueryParams(
-    inputParams
-  );
-  const rows = await fetchCompilerDataFromDb(queryParams);
-  if (rows.length === 0) {
-    return emptyTimeSeriesResponse();
-  }
-  return toCompilerResponseFormat(rows, formats, request_name);
-}
-
 /**
  * backend method to get time series data
  */
@@ -84,47 +58,15 @@ export function toCompilerResponseFormat(
 }
 
 export async function getCompilerCommits(
-  inputparams: any
+  inputParams: any
 ): Promise<CommitResult> {
-  if (!inputparams.startTime || !inputparams.stopTime) {
+  if (!inputParams.startTime || !inputParams.stopTime) {
     throw new Error("no start/end time provided in request");
   }
-  const queryParams = {
-    ...defaultListCommitsInputs, // base defaults
-    ...inputparams, // override with caller's values
-  };
-
-  const arch_list = toQueryArch(inputparams.device, inputparams.arch);
-  queryParams["arch"] = arch_list;
   return await getCommitsWithSampling(
     COMPILER_BENCHMARK_COMMITS_TABLE_NAME,
-    queryParams
-  );
-}
-
-// TODO(ELAINEWY): add GET BENCHMARK DATA API
-async function getSingleCompilerBenchmarkDataQueryParams(
-  inputparams: any
-): Promise<any> {
-  const queryParams = {
-    ...defaultCompilerGetBenchmarkDataInputs, // base defaults
-    ...inputparams, // override with caller's values
-  };
-  const arch_list = toQueryArch(queryParams.device, queryParams.arch);
-  queryParams["arch"] = arch_list;
-
-  if (!queryParams.workflow || !queryParams.branch) {
-    throw new Error(
-      "no workflow or branch provided in request for single data fetch"
-    );
-  }
-  queryParams["workflows"] = [queryParams.workflow];
-  queryParams["branches"] = [queryParams.branch];
-  console.log(
-    "(getSingleCompilerBenchmarkDataQueryParams) workflows provided in request",
-    queryParams.workflows
+    inputParams
   );
-  return queryParams;
 }
 
 /**
@@ -140,10 +82,6 @@ export async function getCompilerBenchmarkTimeRangeQueryParams(
     ...defaultCompilerGetTimeSeriesInputs, // base defaults
     ...inputparams, // override with caller's values
   };
-
-  const arch_list = toQueryArch(queryParams.device, queryParams.arch);
-  queryParams["arch"] = arch_list;
-
   // todo(elainewy): support lworkfow and rworkflow in the future for time range query
 
   // use the startTime and endTime to fetch commits from clickhouse if commits field is not provided
@@ -152,7 +90,6 @@ export async function getCompilerBenchmarkTimeRangeQueryParams(
       "(getCompilerBenchmarkTimeRangeQueryParams) no start/end time provided in request"
     );
   }
-
   if (!queryParams.workflows || queryParams.workflows.length == 0) {
     const { data: commit_results } = await getCommitsWithSampling(
       COMPILER_BENCHMARK_COMMITS_TABLE_NAME,
@@ -188,10 +125,9 @@ async function fetchCompilerDataFromDb(queryParams: any): Promise<any[]> {
   const start = Date.now();
   let rows: any[] = [];
   try {
-    rows = await queryClickhouseSaved(
-      COMPILER_BENCHMARK_TABLE_NAME,
-      queryParams
-    );
+    const fetcher = new BenchmarkCompilerBenchmarkDataQueryBuilder();
+    const data = await fetcher.applyQuery(queryParams);
+    rows = fetcher.postProcess(data);
   } catch (err: any) {
     throw Error(
       `${COMPILER_BENCHMARK_TABLE_NAME}(clickhouse query issue) ${err.message}`

torchci/lib/benchmark/api_helper/backend/compilers/helpers/common.ts

@@ -14,21 +14,6 @@ export function extractBackendSqlStyle(
   return m ? m[1] : null;
 }
 
-export function toQueryArch(device: string, arch: string) {
-  if (arch === undefined) return [];
-  if (!device) return [];
-  switch (device) {
-    case "rocm":
-      if (arch === "mi300x" || arch == "") return ["mi300x", "mi325x"];
-      return [arch];
-    default:
-      if (arch === "") {
-        return [];
-      }
-      return [arch];
-  }
-}
-
 export function toApiArch(device: string, arch: string): string {
   const norm = arch.toLowerCase();
   switch (device) {