Rebase colltrace for v2.28

Summary:
First change to rebase colltrace onto NCCL v2.28. Currently includes both new and old colltrace.
There will be a second diff that will remove the old colltrace invocations.

~~Note: I am running into issues with NCCL 2.28 where it is failing when trying to use ncclGin plugin. For now I have disabled the plugin, but need some feedback on best way to handle the bug in NCCL baselin~~e.
Update: After adding NCCL_GIN_TYPE=0 in cvars, this issue is resolved.

Reviewed By: zhiyongww

Differential Revision: D86152808

fbshipit-source-id: 762bd16f0571b0d066cdf822b7b1e3fc5f0a616b

Author: SuhitK

comms/utils/cvars/nccl_cvars.cc

@@ -1,5 +1,5 @@
 // Copyright (c) Meta Platforms, Inc. and affiliates.
-// @generated SignedSource<<c2b0347281b5f4c58bca1e3b1547530d8f7507e8>>
+// @generated SignedSource<<12578d0c7386c85de2417d28efac7a19e1861246>>
 // Automatically generated by ./comms/utils/cvars/extractcvars.py --- START
 // DO NOT EDIT!!!
 
@@ -680,6 +680,8 @@ int64_t NCCL_GDRCOPY_SYNC_ENABLE;
 int64_t NCCL_GDRCOPY_SYNC_ENABLE_DEFAULT;
 int64_t NCCL_GDR_FLUSH_DISABLE;
 int64_t NCCL_GDR_FLUSH_DISABLE_DEFAULT;
+int64_t NCCL_GIN_TYPE;
+int64_t NCCL_GIN_TYPE_DEFAULT;
 std::string NCCL_GRAPH_DUMP_FILE;
 std::string NCCL_GRAPH_DUMP_FILE_DEFAULT;
 int64_t NCCL_GRAPH_DUMP_FILE_RANK;
@@ -1173,6 +1175,7 @@ std::unordered_map<std::string, int64_t*> env_int64_values = {
     {"NCCL_GDRCOPY_FLUSH_ENABLE", &NCCL_GDRCOPY_FLUSH_ENABLE},
     {"NCCL_GDRCOPY_SYNC_ENABLE", &NCCL_GDRCOPY_SYNC_ENABLE},
     {"NCCL_GDR_FLUSH_DISABLE", &NCCL_GDR_FLUSH_DISABLE},
+    {"NCCL_GIN_TYPE", &NCCL_GIN_TYPE},
     {"NCCL_GRAPH_DUMP_FILE_RANK", &NCCL_GRAPH_DUMP_FILE_RANK},
     {"NCCL_GRAPH_HELPER_DISABLE", &NCCL_GRAPH_HELPER_DISABLE},
     {"NCCL_GRAPH_MIXING_SUPPORT", &NCCL_GRAPH_MIXING_SUPPORT},
@@ -1706,6 +1709,7 @@ static void initEnvSet(std::unordered_set<std::string>& env) {
   env.insert("NCCL_GDRCOPY_FLUSH_ENABLE");
   env.insert("NCCL_GDRCOPY_SYNC_ENABLE");
   env.insert("NCCL_GDR_FLUSH_DISABLE");
+  env.insert("NCCL_GIN_TYPE");
   env.insert("NCCL_GRAPH_DUMP_FILE");
   env.insert("NCCL_GRAPH_DUMP_FILE_RANK");
   env.insert("NCCL_GRAPH_FILE");
@@ -3994,6 +3998,12 @@ static void readCvarEnv() {
     CVAR_INFO(
         "NCCL Config - CVAR {} has an override", "NCCL_GDR_FLUSH_DISABLE");
   }
+  NCCL_GIN_TYPE = env2num<int64_t>("NCCL_GIN_TYPE", "0");
+  NCCL_GIN_TYPE_DEFAULT = env2num<int64_t>("NCCL_ENV_DO_NOT_SET", "0");
+
+  if (NCCL_GIN_TYPE_DEFAULT != NCCL_GIN_TYPE) {
+    CVAR_INFO("NCCL Config - CVAR {} has an override", "NCCL_GIN_TYPE");
+  }
   NCCL_GRAPH_DUMP_FILE = env2str("NCCL_GRAPH_DUMP_FILE", "");
   NCCL_GRAPH_DUMP_FILE_DEFAULT = env2str("NCCL_ENV_DO_NOT_SET", "");
 

comms/utils/cvars/nccl_cvars.h

@@ -1,5 +1,5 @@
 // Copyright (c) Meta Platforms, Inc. and affiliates.
-// @generated SignedSource<<c2b0347281b5f4c58bca1e3b1547530d8f7507e8>>
+// @generated SignedSource<<12578d0c7386c85de2417d28efac7a19e1861246>>
 // Automatically generated by ./comms/utils/cvars/extractcvars.py --- START
 // DO NOT EDIT!!!
 
@@ -677,6 +677,9 @@ extern int64_t NCCL_GDRCOPY_SYNC_ENABLE_DEFAULT;
 extern int64_t NCCL_GDR_FLUSH_DISABLE;
 extern int64_t NCCL_GDR_FLUSH_DISABLE_DEFAULT;
 
+extern int64_t NCCL_GIN_TYPE;
+extern int64_t NCCL_GIN_TYPE_DEFAULT;
+
 extern std::string NCCL_GRAPH_DUMP_FILE;
 extern std::string NCCL_GRAPH_DUMP_FILE_DEFAULT;
 

comms/utils/cvars/nccl_cvars.yaml

@@ -2245,6 +2245,13 @@ cvars:
      We are targeting 8000 ranks most, so leaving room for 100K events should
      be sufficient.
 
+ - name        : NCCL_GIN_TYPE
+   type        : int64_t
+   default     : 0
+   description : |-
+     GPU-Initiated Network (GIN) type configuration. This controls the type
+     of GIN implementation to use for network operations.
+
  - name        : NCCL_PROXYMOCK_NET_SEND_FAILURE
    type        : stringlist
    default     :