Skip to content

Commit 4e06858

Browse files
iddqdexiddqdex
authored
Revert "build with custom graph (#28084)" (#28570)
1 parent 8721579 commit 4e06858

File tree

3 files changed

+75
-205
lines changed

3 files changed

+75
-205
lines changed

‎.github/actions/test_ya/action.yml‎

Lines changed: 18 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -254,18 +254,13 @@ runs:
254254
echo "::debug::get version"
255255
./ya --version
256256
257+
export YA_MAKE_COMMAND="./ya make ${params[@]}"
257258
if [ "${{ inputs.increment }}" = "true" ]; then
258259
GRAPH_COMPARE_OUTPUT="$PUBLIC_DIR/graph_compare_log.txt"
259260
GRAPH_COMPARE_OUTPUT_URL="$PUBLIC_DIR_URL/graph_compare_log.txt"
260261
261262
set +e
262-
YA_MAKE_COMMAND="./ya make ${params[@]}"
263-
if [ true = ${{ inputs.run_tests }} ]; then
264-
YA_MAKE_COMMAND="$YA_MAKE_COMMAND -A"
265-
fi
266-
GRAPH_PATH=$(realpath graph.json)
267-
CONTEXT_PATH=$(realpath context.json)
268-
./.github/scripts/graph_compare.py --ya-make-command="$YA_MAKE_COMMAND" --result-graph-path=$GRAPH_PATH --result-context-path=$CONTEXT_PATH $ORIGINAL_HEAD~1 $ORIGINAL_HEAD |& tee $GRAPH_COMPARE_OUTPUT
263+
./.github/scripts/graph_compare.py $ORIGINAL_HEAD~1 $ORIGINAL_HEAD |& tee $GRAPH_COMPARE_OUTPUT
269264
RC=${PIPESTATUS[0]}
270265
set -e
271266
@@ -278,11 +273,7 @@ runs:
278273
fi
279274
280275
git checkout $ORIGINAL_HEAD
281-
params+=(
282-
--build-custom-json=$GRAPH_PATH
283-
--custom-context=$CONTEXT_PATH
284-
)
285-
YA_MAKE_TARGET="ydb"
276+
YA_MAKE_TARGET=.
286277
else
287278
YA_MAKE_TARGET=""
288279
for TARGET in ${{ inputs.build_target }}; do
@@ -322,11 +313,24 @@ runs:
322313
# Check YDB configuration (after checkout from main)
323314
python3 .github/scripts/check_ydb_qa_config.py || true
324315
316+
if [ ! -z "${{ inputs.bazel_remote_uri }}" ]; then
317+
params+=(--bazel-remote-store)
318+
params+=(--bazel-remote-base-uri "${{ inputs.bazel_remote_uri }}")
319+
fi
320+
321+
if [ "${{ inputs.put_build_results_to_cache }}" = "true" ]; then
322+
params+=(--bazel-remote-username "${{ inputs.bazel_remote_username }}")
323+
params+=(--bazel-remote-password-file "$BAZEL_REMOTE_PASSWORD_FILE")
324+
params+=(--bazel-remote-put --dist-cache-max-file-size=209715200)
325+
fi
326+
325327
if [ true = ${{ inputs.run_tests }} ]; then
326328
params+=(-A)
327329
params+=(--retest)
328330
fi
329331
332+
export YA_MAKE_COMMAND="./ya make ${params[@]}"
333+
330334
YA_MAKE_OUT_DIR=$TMP_DIR/out
331335
332336
YA_MAKE_OUTPUT="$PUBLIC_DIR/ya_make_output.txt"
@@ -352,27 +356,12 @@ runs:
352356
mkdir $TEST_META_INFO
353357
354358
CURRENT_MESSAGE="ya make is running..."
355-
CACHE_OPTS=()
356359
if [ $IS_RETRY = 0 ]; then
357360
CURRENT_MESSAGE="$CURRENT_MESSAGE"
358361
RERUN_FAILED_OPT=""
359-
if [ ! -z "${{ inputs.bazel_remote_uri }}" ]; then
360-
CACHE_OPTS+=(--bazel-remote-store)
361-
CACHE_OPTS+=(--bazel-remote-base-uri "${{ inputs.bazel_remote_uri }}")
362-
fi
363-
364-
if [ "${{ inputs.put_build_results_to_cache }}" = "true" ]; then
365-
CACHE_OPTS+=(--bazel-remote-username "${{ inputs.bazel_remote_username }}")
366-
CACHE_OPTS+=(--bazel-remote-password-file "$BAZEL_REMOTE_PASSWORD_FILE")
367-
CACHE_OPTS+=(--bazel-remote-put --dist-cache-max-file-size=209715200)
368-
fi
369362
else
370363
CURRENT_MESSAGE="$CURRENT_MESSAGE (failed tests rerun, try $RETRY)"
371-
if [ -z "$GRAPH_PATH" ]; then
372-
RERUN_FAILED_OPT="-X"
373-
else
374-
RERUN_FAILED_OPT=""
375-
fi
364+
RERUN_FAILED_OPT="-X"
376365
fi
377366
378367
echo $CURRENT_MESSAGE | GITHUB_TOKEN="${{ github.token }}" .github/scripts/tests/comment-pr.py
@@ -394,7 +383,7 @@ runs:
394383
MONITOR_PID=$!
395384
396385
set +e
397-
(./ya make ${params[@]} ${CACHE_OPTS[@]} $YA_MAKE_TARGET \
386+
($YA_MAKE_COMMAND $YA_MAKE_TARGET \
398387
$RERUN_FAILED_OPT --log-file "$PUBLIC_DIR/ya_log.txt" \
399388
--evlog-file "$CURRENT_PUBLIC_DIR/ya_evlog.jsonl" \
400389
--junit "$CURRENT_JUNIT_XML_PATH" --build-results-report "$CURRENT_REPORT" --output "$YA_MAKE_OUT_DIR"; echo $? > exit_code) |& cat >> $YA_MAKE_OUTPUT
@@ -536,16 +525,6 @@ runs:
536525
if [ $IS_LAST_RETRY = 1 ]; then
537526
break
538527
fi
539-
if [ -n "$GRAPH_PATH" ] && [ -n "$CONTEXT_PATH" ]; then
540-
NEW_GRAPH_PATH=$(realpath "graph-$RETRY.json")
541-
NEW_CONTEXT_PATH=$(realpath "context-$RETRY.json")
542-
.github/scripts/graph_patch.py \
543-
--in-graph "$GRAPH_PATH" --in-context "$CONTEXT_PATH" \
544-
--out-graph "$NEW_GRAPH_PATH" --out-context "$NEW_GRAPH_PATH" \
545-
--report="$CURRENT_REPORT"
546-
GRAPH_PATH=$NEW_GRAPH_PATH
547-
CONTEXT_PATH=$NEW_CONTEXT_PATH
548-
fi
549528
done;
550529
551530
if [ $BUILD_FAILED = 0 ]; then

‎.github/scripts/graph_compare.py‎

Lines changed: 57 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@
66

77
import os
88
import tempfile
9-
import argparse
9+
import sys
10+
import json
1011

1112

1213
def exec(command: str):
@@ -21,8 +22,17 @@ def log(msg: str):
2122
print(msg)
2223

2324

24-
def main(ya_make_command: str, graph_path: str, context_path: str, base_commit: str, head_commit: str) -> None:
25-
ya = ya_make_command.split(' ')[0]
25+
def do_compare():
26+
if len(sys.argv) < 3:
27+
print('base or head commit not set')
28+
exit(1)
29+
base_commit = sys.argv[1]
30+
head_commit = sys.argv[2]
31+
32+
ya_make_command = os.getenv('YA_MAKE_COMMAND')
33+
if not ya_make_command:
34+
print('YA_MAKE_COMMAND not set')
35+
exit(1)
2636

2737
workdir = os.getenv('workdir')
2838
if not workdir:
@@ -32,41 +42,57 @@ def main(ya_make_command: str, graph_path: str, context_path: str, base_commit:
3242
log('Checkout base commit...')
3343
exec(f'git checkout {base_commit}')
3444
log('Build graph for base commit...')
35-
exec(f'{ya_make_command} ydb -k --cache-tests --save-graph-to {workdir}/graph_base.json --save-context-to {workdir}/context_base.json')
45+
exec(f'{ya_make_command} ydb -k -A --cache-tests -Gj0 > {workdir}/graph_base.json')
3646

3747
log('Checkout head commit...')
3848
exec(f'git checkout {head_commit}')
3949
log('Build graph for head commit...')
40-
exec(f'{ya_make_command} ydb -k --cache-tests --save-graph-to {workdir}/graph_head.json --save-context-to {workdir}/context_head.json')
50+
exec(f'{ya_make_command} ydb -k -A --cache-tests -Gj0 > {workdir}/graph_head.json')
4151

4252
log('Generate diff graph...')
43-
exec(f'{ya} tool ygdiff --old {workdir}/graph_base.json --new {workdir}/graph_head.json --cut {graph_path} --dump-uids {workdir}/uids.json --no-cache-for-affected-nodes')
53+
exec(f'./ya tool ygdiff --old {workdir}/graph_base.json --new {workdir}/graph_head.json --cut {workdir}/graph_diff.json --dump-uids-for-affected-nodes {workdir}/affected_uids.json')
54+
55+
log('Read diff graph...')
56+
with open(f'{workdir}/graph_diff.json', 'r') as f:
57+
diff_graph = json.load(f)
58+
59+
with open(f'{workdir}/affected_uids.json', 'r') as f:
60+
uids = set(json.load(f))
61+
62+
tests = set()
63+
modules = set()
64+
65+
log('Scan diff graph...')
66+
for target in diff_graph.get('graph', []):
67+
if target.get('uid') not in uids:
68+
continue
69+
if target.get('node-type') == 'test':
70+
path = target.get('kv', {}).get('path')
71+
if path is not None:
72+
tests.add(os.path.dirname(path))
73+
tp = target.get('target_properties')
74+
if (
75+
tp is not None
76+
and tp.get('module_type') is not None
77+
and tp.get('module_dir', '').startswith('ydb')
78+
and tp.get('module_tag', '').find('proto') < 0
79+
):
80+
modules.add(tp.get('module_dir'))
81+
82+
log('Create ya.make')
4483

45-
log('Generate diff context...')
46-
exec(f'{ya} tool context_difference {workdir}/context_base.json {workdir}/context_head.json {context_path} {workdir}/uids.json {graph_path}')
84+
with open('ya.make', 'w') as ya_make:
85+
ya_make.write('RECURSE_FOR_TESTS(\n')
86+
for test in sorted(tests):
87+
ya_make.write(f' {test}\n')
88+
ya_make.write(')\n\nRECURSE (\n')
89+
for module in sorted(modules):
90+
ya_make.write(f' {module}\n')
91+
ya_make.write(')\n')
92+
log('ya.make content:')
93+
exec('cat ya.make')
94+
exit(0)
4795

4896

4997
if __name__ == '__main__':
50-
parser = argparse.ArgumentParser()
51-
parser.add_argument(
52-
'--result-graph-path', '-g', type=str, dest='result_graph_path', required=True,
53-
help='Path to result graph'
54-
)
55-
parser.add_argument(
56-
'--result-context-path', '-c', type=str, dest='result_context_path', required=True,
57-
help='Path to result context'
58-
)
59-
parser.add_argument(
60-
'--ya-make-command', '-y', type=str, dest='ya_make_command', required=True,
61-
help='Ya make command'
62-
)
63-
parser.add_argument(dest='base_commit', help='Base commit')
64-
parser.add_argument(dest='head_commit', help='Head commit')
65-
opts = parser.parse_args()
66-
main(
67-
ya_make_command=opts.ya_make_command,
68-
graph_path=opts.result_graph_path,
69-
context_path=opts.result_context_path,
70-
base_commit=opts.base_commit,
71-
head_commit=opts.head_commit
72-
)
98+
do_compare()

‎.github/scripts/graph_patch.py‎

Lines changed: 0 additions & 135 deletions
This file was deleted.

0 commit comments

Comments
 (0)