Merge #39

39: Update libFuzzer to be from the 'release_90' branch r=nagisa a=alex



Co-authored-by: Alex Gaynor <alex.gaynor@gmail.com>

Author: bors[bot]

.travis.yml

@@ -1,7 +1,7 @@
 language: rust
 sudo: false
 cache: cargo
-dist: trusty
+dist: bionic
 rust:
   - nightly
 os:
@@ -12,8 +12,8 @@ notifications:
   email: false
 script:
   - cd example
-  - cargo rustc --release -- -Cpasses='sancov' -Cllvm-args=-sanitizer-coverage-level=4 -Cllvm-args=-sanitizer-coverage-trace-compares -Cpanic=abort -Cllvm-args=-sanitizer-coverage-trace-divs -Cllvm-args=-sanitizer-coverage-trace-geps -Cllvm-args=-sanitizer-coverage-prune-blocks=0 -Zsanitizer=address
+  - cargo rustc --release -- -Cpasses='sancov' -Cllvm-args=-sanitizer-coverage-level=4 -Cllvm-args=-sanitizer-coverage-trace-compares -Cllvm-args=-sanitizer-coverage-inline-8bit-counters -Cllvm-args=-sanitizer-coverage-stack-depth -Cllvm-args=-sanitizer-coverage-trace-geps -Cllvm-args=-sanitizer-coverage-prune-blocks=0 -Zsanitizer=address
   - (! ./target/release/example -runs=100000)
   - cd ../example_arbitrary
-  - cargo rustc --release -- -Cpasses='sancov' -Cllvm-args=-sanitizer-coverage-level=4 -Cllvm-args=-sanitizer-coverage-trace-compares -Cpanic=abort -Cllvm-args=-sanitizer-coverage-trace-divs -Cllvm-args=-sanitizer-coverage-trace-geps -Cllvm-args=-sanitizer-coverage-prune-blocks=0 -Zsanitizer=address
+  - cargo rustc --release -- -Cpasses='sancov' -Cllvm-args=-sanitizer-coverage-level=4 -Cllvm-args=-sanitizer-coverage-trace-compares -Cllvm-args=-sanitizer-coverage-inline-8bit-counters -Cllvm-args=-sanitizer-coverage-stack-depth -Cllvm-args=-sanitizer-coverage-trace-geps -Cllvm-args=-sanitizer-coverage-prune-blocks=0 -Zsanitizer=address
   - (! ./target/release/example -runs=10000000)

libfuzzer/CMakeLists.txt

@@ -3,19 +3,17 @@ set(LIBFUZZER_SOURCES
   FuzzerDataFlowTrace.cpp
   FuzzerDriver.cpp
   FuzzerExtFunctionsDlsym.cpp
-  FuzzerExtFunctionsDlsymWin.cpp
   FuzzerExtFunctionsWeak.cpp
+  FuzzerExtFunctionsWindows.cpp
   FuzzerExtraCounters.cpp
+  FuzzerFork.cpp
   FuzzerIO.cpp
   FuzzerIOPosix.cpp
   FuzzerIOWindows.cpp
   FuzzerLoop.cpp
   FuzzerMerge.cpp
   FuzzerMutate.cpp
   FuzzerSHA1.cpp
-  FuzzerShmemFuchsia.cpp
-  FuzzerShmemPosix.cpp
-  FuzzerShmemWindows.cpp
   FuzzerTracePC.cpp
   FuzzerUtil.cpp
   FuzzerUtilDarwin.cpp
@@ -25,6 +23,8 @@ set(LIBFUZZER_SOURCES
   FuzzerUtilWindows.cpp)
 
 set(LIBFUZZER_HEADERS
+  FuzzerBuiltins.h
+  FuzzerBuiltinsMsvc.h
   FuzzerCommand.h
   FuzzerCorpus.h
   FuzzerDataFlowTrace.h
@@ -33,6 +33,7 @@ set(LIBFUZZER_HEADERS
   FuzzerExtFunctions.def
   FuzzerExtFunctions.h
   FuzzerFlags.def
+  FuzzerFork.h
   FuzzerIO.h
   FuzzerInterface.h
   FuzzerInternal.h
@@ -41,7 +42,6 @@ set(LIBFUZZER_HEADERS
   FuzzerOptions.h
   FuzzerRandom.h
   FuzzerSHA1.h
-  FuzzerShmem.h
   FuzzerTracePC.h
   FuzzerUtil.h
   FuzzerValueBitMap.h)
@@ -55,7 +55,9 @@ CHECK_CXX_SOURCE_COMPILES("
 
 set(LIBFUZZER_CFLAGS ${SANITIZER_COMMON_CFLAGS})
 
-if(OS_NAME MATCHES "Linux|Fuchsia" AND COMPILER_RT_LIBCXX_PATH)
+if(OS_NAME MATCHES "Linux|Fuchsia" AND
+   COMPILER_RT_LIBCXX_PATH AND
+   COMPILER_RT_LIBCXXABI_PATH)
   list(APPEND LIBFUZZER_CFLAGS -nostdinc++ -D_LIBCPP_ABI_VERSION=Fuzzer)
   # Remove -stdlib= which is unused when passing -nostdinc++.
   string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
@@ -69,12 +71,21 @@ if (CMAKE_CXX_FLAGS MATCHES "fsanitize-coverage")
   list(APPEND LIBFUZZER_CFLAGS -fno-sanitize-coverage=trace-pc-guard,edge,trace-cmp,indirect-calls,8bit-counters)
 endif()
 
-if(NOT HAS_THREAD_LOCAL)
-  list(APPEND LIBFUZZER_CFLAGS -Dthread_local=__thread)
+if(MSVC)
+  # Silence warnings by turning off exceptions in MSVC headers and avoid an
+  # error by unecessarily defining thread_local when it isn't even used on
+  # Windows.
+  list(APPEND LIBFUZZER_CFLAGS -D_HAS_EXCEPTIONS=0)
+else()
+  if(NOT HAS_THREAD_LOCAL)
+    list(APPEND LIBFUZZER_CFLAGS -Dthread_local=__thread)
+  endif()
 endif()
 
 set(FUZZER_SUPPORTED_OS ${SANITIZER_COMMON_SUPPORTED_OS})
 
+add_compiler_rt_component(fuzzer)
+
 add_compiler_rt_object_libraries(RTfuzzer
   OS ${FUZZER_SUPPORTED_OS}
   ARCHS ${FUZZER_SUPPORTED_ARCH}
@@ -106,12 +117,19 @@ add_compiler_rt_runtime(clang_rt.fuzzer_no_main
   CFLAGS ${LIBFUZZER_CFLAGS}
   PARENT_TARGET fuzzer)
 
-if(OS_NAME MATCHES "Linux|Fuchsia" AND COMPILER_RT_LIBCXX_PATH)
+if(OS_NAME MATCHES "Linux|Fuchsia" AND
+   COMPILER_RT_LIBCXX_PATH AND
+   COMPILER_RT_LIBCXXABI_PATH)
   macro(partially_link_libcxx name dir arch)
+    if(${arch} MATCHES "i386")
+      set(EMULATION_ARGUMENT "-m" "elf_i386")
+    else()
+      set(EMULATION_ARGUMENT "")
+    endif()
     set(cxx_${arch}_merge_dir "${CMAKE_CURRENT_BINARY_DIR}/cxx_${arch}_merge.dir")
     file(MAKE_DIRECTORY ${cxx_${arch}_merge_dir})
     add_custom_command(TARGET clang_rt.${name}-${arch} POST_BUILD
-      COMMAND ${CMAKE_LINKER} --whole-archive "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" --no-whole-archive ${dir}/lib/libc++.a -r -o ${name}.o
+      COMMAND ${CMAKE_LINKER} ${EMULATION_ARGUMENT} --whole-archive "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" --no-whole-archive ${dir}/lib/libc++.a -r -o ${name}.o
       COMMAND ${CMAKE_OBJCOPY} --localize-hidden ${name}.o
       COMMAND ${CMAKE_COMMAND} -E remove "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>"
       COMMAND ${CMAKE_AR} qcs "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" ${name}.o
@@ -124,13 +142,8 @@ if(OS_NAME MATCHES "Linux|Fuchsia" AND COMPILER_RT_LIBCXX_PATH)
     set(LIBCXX_${arch}_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/libcxx_fuzzer_${arch})
     add_custom_libcxx(libcxx_fuzzer_${arch} ${LIBCXX_${arch}_PREFIX}
       CFLAGS ${TARGET_CFLAGS}
-             -D_LIBCPP_ABI_VERSION=Fuzzer
-             -D_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS=1
-             -fvisibility=hidden
       CMAKE_ARGS -DCMAKE_CXX_COMPILER_WORKS=ON
-                 -DLIBCXX_ENABLE_EXCEPTIONS=OFF
-                 -DLIBCXX_ENABLE_SHARED=OFF
-                 -DLIBCXX_CXX_ABI=none)
+                 -DLIBCXX_ABI_NAMESPACE=Fuzzer)
     target_compile_options(RTfuzzer.${arch} PRIVATE -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1)
     add_dependencies(RTfuzzer.${arch} libcxx_fuzzer_${arch}-build)
     target_compile_options(RTfuzzer_main.${arch} PRIVATE -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1)

libfuzzer/FuzzerBuiltins.h

@@ -0,0 +1,35 @@
+//===- FuzzerBuiltins.h - Internal header for builtins ----------*- C++ -* ===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Wrapper functions and marcos around builtin functions.
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FUZZER_BUILTINS_H
+#define LLVM_FUZZER_BUILTINS_H
+
+#include "FuzzerDefs.h"
+
+#if !LIBFUZZER_MSVC
+#include <cstdint>
+
+#define GET_CALLER_PC() __builtin_return_address(0)
+
+namespace fuzzer {
+
+inline uint8_t  Bswap(uint8_t x)  { return x; }
+inline uint16_t Bswap(uint16_t x) { return __builtin_bswap16(x); }
+inline uint32_t Bswap(uint32_t x) { return __builtin_bswap32(x); }
+inline uint64_t Bswap(uint64_t x) { return __builtin_bswap64(x); }
+
+inline uint32_t Clzll(unsigned long long X) { return __builtin_clzll(X); }
+inline uint32_t Clz(unsigned long long X) { return __builtin_clz(X); }
+inline int Popcountll(unsigned long long X) { return __builtin_popcountll(X); }
+
+}  // namespace fuzzer
+
+#endif  // !LIBFUZZER_MSVC
+#endif  // LLVM_FUZZER_BUILTINS_H

libfuzzer/FuzzerBuiltinsMsvc.h

@@ -0,0 +1,58 @@
+//===- FuzzerBuiltinsMSVC.h - Internal header for builtins ------*- C++ -* ===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Wrapper functions and marcos that use intrinsics instead of builtin functions
+// which cannot be compiled by MSVC.
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FUZZER_BUILTINS_MSVC_H
+#define LLVM_FUZZER_BUILTINS_MSVC_H
+
+#include "FuzzerDefs.h"
+
+#if LIBFUZZER_MSVC
+#if !defined(_M_ARM) && !defined(_M_X64)
+#error "_BitScanReverse64 unavailable on this platform so MSVC is unsupported."
+#endif
+#include <intrin.h>
+#include <cstdint>
+#include <cstdlib>
+
+// __builtin_return_address() cannot be compiled with MSVC. Use the equivalent
+// from <intrin.h>
+#define GET_CALLER_PC() _ReturnAddress()
+
+namespace fuzzer {
+
+inline uint8_t  Bswap(uint8_t x)  { return x; }
+// Use alternatives to __builtin functions from <stdlib.h> and <intrin.h> on
+// Windows since the builtins are not supported by MSVC.
+inline uint16_t Bswap(uint16_t x) { return _byteswap_ushort(x); }
+inline uint32_t Bswap(uint32_t x) { return _byteswap_ulong(x); }
+inline uint64_t Bswap(uint64_t x) { return _byteswap_uint64(x); }
+
+// The functions below were mostly copied from
+// compiler-rt/lib/builtins/int_lib.h which defines the __builtin functions used
+// outside of Windows.
+inline uint32_t Clzll(uint64_t X) {
+  unsigned long LeadZeroIdx = 0;
+  if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx;
+  return 64;
+}
+
+inline uint32_t Clz(uint32_t X) {
+  unsigned long LeadZeroIdx = 0;
+  if (_BitScanReverse(&LeadZeroIdx, X)) return 31 - LeadZeroIdx;
+  return 32;
+}
+
+inline int Popcountll(unsigned long long X) { return __popcnt64(X); }
+
+}  // namespace fuzzer
+
+#endif  // LIBFUZER_MSVC
+#endif  // LLVM_FUZZER_BUILTINS_MSVC_H

libfuzzer/FuzzerCommand.h

@@ -1,9 +1,8 @@
 //===- FuzzerCommand.h - Interface representing a process -------*- C++ -* ===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // FuzzerCommand represents a command to run in a subprocess.  It allows callers
@@ -81,7 +80,7 @@ class Command final {
   }
 
   // Like hasArgument, but checks for "-[Flag]=...".
-  bool hasFlag(const std::string &Flag) {
+  bool hasFlag(const std::string &Flag) const {
     std::string Arg("-" + Flag + "=");
     auto IsMatch = [&](const std::string &Other) {
       return Arg.compare(0, std::string::npos, Other, 0, Arg.length()) == 0;
@@ -92,7 +91,7 @@ class Command final {
   // Returns the value of the first instance of a given flag, or an empty string
   // if the flag isn't present.  Ignores any occurrences after
   // "-ignore_remaining_args=1", if present.
-  std::string getFlagValue(const std::string &Flag) {
+  std::string getFlagValue(const std::string &Flag) const {
     std::string Arg("-" + Flag + "=");
     auto IsMatch = [&](const std::string &Other) {
       return Arg.compare(0, std::string::npos, Other, 0, Arg.length()) == 0;

libfuzzer/FuzzerCorpus.h

@@ -1,9 +1,8 @@
 //===- FuzzerCorpus.h - Internal header for the Fuzzer ----------*- C++ -* ===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // fuzzer::InputCorpus
@@ -86,9 +85,10 @@ class InputCorpus {
 
   bool empty() const { return Inputs.empty(); }
   const Unit &operator[] (size_t Idx) const { return Inputs[Idx]->U; }
-  void AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile,
-                   bool HasFocusFunction, const Vector<uint32_t> &FeatureSet,
-                   const DataFlowTrace &DFT, const InputInfo *BaseII) {
+  InputInfo *AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile,
+                         bool HasFocusFunction,
+                         const Vector<uint32_t> &FeatureSet,
+                         const DataFlowTrace &DFT, const InputInfo *BaseII) {
     assert(!U.empty());
     if (FeatureDebug)
       Printf("ADD_TO_CORPUS %zd NF %zd\n", Inputs.size(), NumFeatures);
@@ -114,6 +114,7 @@ class InputCorpus {
     UpdateCorpusDistribution();
     PrintCorpus();
     // ValidateFeatureSet();
+    return &II;
   }
 
   // Debug-only
@@ -170,7 +171,7 @@ class InputCorpus {
     InputInfo &II = *Inputs[ChooseUnitIdxToMutate(Rand)];
     assert(!II.U.empty());
     return II;
-  };
+  }
 
   // Returns an index of random unit from the corpus to mutate.
   size_t ChooseUnitIdxToMutate(Random &Rand) {
@@ -238,12 +239,6 @@ class InputCorpus {
     return false;
   }
 
-  bool IsFeatureNew(size_t Idx, uint32_t NewSize, bool Shrink) {
-    assert(NewSize);
-    uint32_t OldSize = GetFeature(Idx % kFeatureSetSize);
-    return OldSize == 0 || (Shrink && OldSize > NewSize);
-  }
-
   size_t NumFeatures() const { return NumAddedFeatures; }
   size_t NumFeatureUpdates() const { return NumUpdatedFeatures; }
 

libfuzzer/FuzzerCrossOver.cpp

@@ -1,9 +1,8 @@
 //===- FuzzerCrossOver.cpp - Cross over two test inputs -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Cross over test inputs.