ci: add flaky test catching workflow

.github/workflows/catch_flaky.yml

@@ -0,0 +1,88 @@
+name: catch_flaky
+
+on:
+  pull_request:
+    paths:
+      - 'test/*/*_test.lua'
+      - 'test/*/*.test.lua'
+
+jobs:
+  catch_flaky:
+    if: github.event_name == 'pull_request' &&
+        !contains(github.event.pull_request.labels.*.name, 'noflaky')
+
+    env:
+      TNT_RELEASE_PATH: /home/runner/tnt-release
+
+    runs-on: ubuntu-22.04
+    timeout-minutes: 180
+    steps:
+      - name: Create variables for Tarantool
+        run: |
+          branch=master
+          commit_hash=$(git ls-remote \
+                        https://github.com/tarantool/tarantool.git \
+                        --branch ${branch} | head -c 8)
+          echo "TNT_BRANCH=${branch}" >> $GITHUB_ENV
+          echo "VERSION_POSTFIX=-${commit_hash}" >> $GITHUB_ENV
+        shell: bash
+
+      - name: Cache tarantool build
+        id: cache-tnt-release
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.TNT_RELEASE_PATH }}
+          key: cache-tnt-release-master${{ env.VERSION_POSTFIX }}
+
+      - name: Clone tarantool
+        if: steps.cache-tnt-release.outputs.cache-hit != 'true'
+        uses: actions/checkout@v3
+        with:
+          repository: tarantool/tarantool
+          ref: ${{ env.TNT_BRANCH }}
+          path: tarantool
+          fetch-depth: 0
+          submodules: true
+
+      - name: Build tarantool
+        if: steps.cache-tnt-release.outputs.cache-hit != 'true'
+        run: |
+          sudo apt-get -y install build-essential cmake make zlib1g-dev \
+            libreadline-dev libncurses5-dev libssl-dev libunwind-dev \
+            libicu-dev python3 python3-yaml python3-six python3-gevent
+          cd ${GITHUB_WORKSPACE}/tarantool
+          mkdir build && cd build
+          cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_DIST=ON
+          make && make DESTDIR=${TNT_RELEASE_PATH} install
+
+      - name: Install tarantool
+        # Workaround as actions/cache cannot restore data to /usr/local
+        run: sudo cp -rvP ${TNT_RELEASE_PATH}/usr/local/* /usr/local/
+
+      - name: Clone the module
+        uses: actions/checkout@v3
+        with:
+          # Fetch the entire history for all branches and tags. It is needed for
+          # upgrade testing.
+          fetch-depth: 0
+          # Enable recursive submodules checkout as test-run git module is used
+          # for running tests.
+          submodules: recursive
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Install test requirements
+        run: pip3 install --user -r test-run/requirements.txt
+
+      - name: Setup tt
+        run: |
+          curl -L https://tarantool.io/release/2/installer.sh | sudo bash
+          sudo apt install -y tt
+          tt version
+
+      - name: Setup luatest
+        run: tt rocks install luatest
+
+      - run: cmake .
+      - run: make test-flaky
+        env:
+          RANGE: HEAD~${{ github.event.pull_request.commits }}..HEAD

.github/workflows/fast_testing.yml

@@ -100,5 +100,14 @@ jobs:
       - name: Install test requirements
         run: pip3 install --user -r test-run/requirements.txt
 
+      - name: Setup tt
+        run: |
+          curl -L https://tarantool.io/release/2/installer.sh | sudo bash
+          sudo apt install -y tt
+          tt version
+
+      - name: Setup luatest
+        run: tt rocks install luatest
+
       - run: cmake .
       - run: make test-force

test/CMakeLists.txt

@@ -8,3 +8,13 @@ add_custom_target(test-force
         --builddir=${PROJECT_BINARY_DIR}
         --vardir=${PROJECT_BINARY_DIR}/test/var
         --force)
+
+add_custom_target(test-flaky
+    COMMAND git diff --relative=test/ --name-only
+        $$\{RANGE:-master..HEAD\}
+        -- "./*/*test.lua" |
+        xargs -L 128 -r ${PROJECT_SOURCE_DIR}/test/test-run.py
+        --force --retries=0
+        --repeat=$$\{N_TRIALS:-32\}
+        --builddir=${PROJECT_BINARY_DIR}
+        --vardir=${PROJECT_BINARY_DIR}/test/var)

test/luatest_helpers/server.lua

@@ -281,7 +281,7 @@ function Server:grep_log(what, bytes, opts)
     local opts = opts or {}
     local noreset = opts.noreset or false
     -- if instance has crashed provide filename to use grep_log
-    local filename = opts.filename or self:eval('return box.cfg.log')
+    local filename = opts.filename or self.log_file
     local file = fio.open(filename, {'O_RDONLY', 'O_NONBLOCK'})
     -- This is needed to find UUID with string.match.
     what = string.gsub(what, "%-", "%%-")