Merge branch 'develop' into tanjinhong72-patch-12

Author: pwseg

CONTRIBUTING.md

@@ -75,7 +75,7 @@ Sources pages check if the source is a cloud-app, then include information about
 
 ## Edit pages
 
-Content with in each `.md` file is markdown. For information about styling, and available extensions, see `_src/utils/formatguide.md` or the live version [here](https://segment.com/docs/utils/formatguide).
+Content with in each `.md` file is markdown. For information about styling, and available extensions, see `_src/utils/formatguide.md` or the live version in the [utils section of the docs](/docs/utils/formatguide).
 
 ## Building a preview
 
@@ -109,6 +109,7 @@ Front matter variables have unique functions, including the following:
 - `hide-boilerplate`: defaults to false. When true, none of the content from `destination-footer.md` is appended to the destination page.
 - `hide-cmodes`: defaults to false. A renaming of "rewrite" for more clarity, hides the connection modes table in the boilerplate.
 - `hide-personas-partial`: defaults to false. When true, hides the section of content from `destination-footer.md` that talks about being able to receive personas data.
+- `hide_actions`: used to hide individual actions. Requires the `id` and `name` of each action.
 - `integration_type`: This is set in the `_config.yml` on three paths to add a noun (Source, Destination, or Warehouse) to the end of the title, and the end of the title tag in the html layout. It also controls the layout and icon for some of these.
 - `source-type`: These are only used to supplement when a Cloud App in the sources path doesn't appear in the Config API list, and needs its type explicitly set. It runs some logic in the `cloud-app-note.md` to explain which cloud-apps are object vs event sources.
 - `private`: Used to indicate that a destination is not publicly available (Private Beta or Pilot status), and is not available in the public catalog. When `private: true`, the build pulls integration metadata from `src/_data/catalog/destinations_private.yml`. To update the list of private destinations, use the `make private_destination` command, and enter the integration's ID when prompted.

docker-compose.yml

@@ -5,8 +5,17 @@ services:
     command: make dev
     image: jekyll/jekyll:latest
     volumes:
-      - '.:/srv/jekyll'
-      - './vendor/bundle:/usr/local/bundle'
+    - '.:/srv/jekyll'
+    - './vendor/bundle:/usr/local/bundle'
     ports:
-      - 4000:4000
+    - 4000:4000
     container_name: jekyll
+    # If you are on arm64 and experiencing issues with the tests (hangs,
+    # connection reset) then try the following in order:
+
+    # - stopping and removing all downloaded container images
+    # - ensuring you have the latest Docker Desktop version
+    # - factory reset your Docker Desktop settings
+
+    # If you are still running into issues please post in #help-infra-seg.
+    platform: linux/amd64

scripts/catalog/updateDestinations.js

@@ -14,6 +14,15 @@ require('dotenv').config();
 
 const PAPI_URL = "https://api.segmentapis.com";
 
+  // Function to remove hidden fields from action
+const removeHiddenFields=function (actions) {
+        return actions.map(action => ({
+            ...action,
+            fields: action.fields.filter(field => !field.hidden)
+        })
+        );
+      }
+
 
 const updateDestinations = async () => {
     let destinations = [];
@@ -88,9 +97,9 @@ const updateDestinations = async () => {
       settings.forEach(setting => {
         setting.description = sanitize(setting.description);
       });
-  
-      let actions = destination.actions;
-      let presets = destination.presets;
+
+     let actions = removeHiddenFields(destination.actions);
+     let presets = destination.presets;
 
       const clone = (obj) => Object.assign({}, obj);
       const renameKey = (object, key, newKey) => {

scripts/catalog/updateSources.js

@@ -14,28 +14,28 @@ const PAPI_URL = "https://api.segmentapis.com";
 
 const regionalSupport = yaml.load(fs.readFileSync(path.resolve(__dirname, `../../src/_data/regional-support.yml`)));
 
-// This file keeps a list of known test sources that show up in the system. 
+// This file keeps a list of known test sources that show up in the system.
 // Because we don't have a status value for sources, they end up showing in our catalog.
 // We use this below to prevent them from being written to yaml.
 const testSources = yaml.load(fs.readFileSync(path.resolve(__dirname, `../../src/_data/catalog/test_sources.yml`)));
 
 
 const updateSources = async () => {
-    let sources = [];                     // Initialize an empty array to hold all sources  
-    let sourcesUpdated = [];              // Initialize an empty array to hold all sources that have been updated    
-    let regionalSourcesUpdated = [];      // Initialize an empty array to hold updated source regional information            
-    let nextPageToken = "MA==";           // Set the initial page token to the first page      
-    let categories = new Set();           // Initialize an empty set to hold all categories      
-    let sourceCategories = [];            // Initialize an empty array to hold all source categories      
-  
-  
+    let sources = [];                     // Initialize an empty array to hold all sources
+    let sourcesUpdated = [];              // Initialize an empty array to hold all sources that have been updated
+    let regionalSourcesUpdated = [];      // Initialize an empty array to hold updated source regional information
+    let nextPageToken = "MA==";           // Set the initial page token to the first page
+    let categories = new Set();           // Initialize an empty set to hold all categories
+    let sourceCategories = [];            // Initialize an empty array to hold all source categories
+
+
     // Get all sources from the catalog
     while (nextPageToken !== undefined) {
       const res = await getCatalog(`${PAPI_URL}/catalog/sources/`, nextPageToken);
       sources = sources.concat(res.data.sourcesCatalog);
       nextPageToken = res.data.pagination.next;
     }
-  
+
     // Sort the sources alphabetically
     sources.sort((a, b) => {
       if (a.name.toLowerCase() < b.name.toLowerCase()) {
@@ -46,7 +46,7 @@ const updateSources = async () => {
       }
       return 0;
     });
-  
+
     // Set the list of categories for libraries
     const libraryCategories = [
       'server',
@@ -55,20 +55,20 @@ const updateSources = async () => {
       'roku',
       'website'
     ];
-  
+
     // Here, define some sources that are real, but that we want to hide.
     const hiddenSources = [
       'amp',
       'factual-engine',
       'twilio-event-streams-beta',
       'ibm-watson-assistant'
     ];
-  
+
     // More regional stuff
     const regionalSourceEndpoint = regionalSupport.sources.endpoint;
     const regionalSourceRegion = regionalSupport.sources.region;
-  
-    
+
+
     // Loop through all sources and create a new object with the data we want
     sources.forEach(source => {
       let slug = slugify(source.name, "sources");
@@ -77,14 +77,14 @@ const updateSources = async () => {
       let regions = ['us'];
       let endpoints = ['us'];
       let mainCategory = source.categories[0] ? source.categories[0].toLowerCase() : '';
-  
+
       if (libraryCategories.includes(mainCategory)) {
         url = `connections/sources/catalog/libraries/${mainCategory}/${slug}`;
       } else {
         url = `connections/sources/catalog/cloud-apps/${slug}`;
         mainCategory = 'cloud-app';
       }
-  
+
       // Sort the settings alphabetically
       settings.sort((a, b) => {
         if (a.name.toLowerCase() < b.name.toLowerCase()) {
@@ -95,19 +95,19 @@ const updateSources = async () => {
         }
         return 0;
       });
-  
+
       if (hiddenSources.includes(slug)) {
         hidden = true;
       }
-  
+
       if (regionalSourceEndpoint.includes(slug)) {
         endpoints.push('eu');
       }
-  
+
       if (regionalSourceRegion.includes(slug)) {
         regions.push('eu');
       }
-  
+
       // If the source ID is in the list of test sources, skip it.
       // If it's not, add it to the list of sources to be written to yaml.
       if (testSources.includes(source.id)) {
@@ -128,13 +128,15 @@ const updateSources = async () => {
             url: source.logos.default
           },
           categories: source.categories,
+          status: source.status,
+          partnerOwned: source.partnerOwned
         };
         sourcesUpdated.push(updatedSource);
         doesCatalogItemExist(updatedSource);
       }
-  
+
       source.categories.reduce((s, e) => s.add(e), categories);
-  
+
       // Sources don't yet have regional information in the Public API, so we write that info here.
       let updatedRegional = {
         id: source.id,
@@ -147,7 +149,7 @@ const updateSources = async () => {
       };
       regionalSourcesUpdated.push(updatedRegional);
     });
-  
+
     const sourceArray = Array.from(categories);
     sourceArray.forEach(category => {
       sourceCategories.push({
@@ -164,36 +166,36 @@ const updateSources = async () => {
         return 0;
       });
     });
-  
+
     const options = {
       noArrayIndent: false
     };
     const todayDate = new Date().toISOString().slice(0, 10);
-  
+
     // Create source catalog YAML file
     let output = "# AUTOGENERATED FROM PUBLIC API. DO NOT EDIT\n";
     output += "# sources last updated " + todayDate + " \n";
     output += yaml.dump({
       items: sourcesUpdated
     }, options);
     fs.writeFileSync(path.resolve(__dirname, `../../src/_data/catalog/sources.yml`), output);
-  
+
     // Create source-category mapping YAML file
     output = "# AUTOGENERATED FROM PUBLIC API. DO NOT EDIT\n";
     output += "# source categories last updated " + todayDate + " \n";
     output += yaml.dump({
       items: sourceCategories
     }, options);
     fs.writeFileSync(path.resolve(__dirname, `../../src/_data/catalog/source_categories.yml`), output);
-  
+
     // Create regional support YAML file
     output = yaml.dump({
       sources: regionalSourcesUpdated
     }, options);
     fs.writeFileSync(path.resolve(__dirname, `../../src/_data/catalog/regional-supported.yml`), output);
-  
+
     console.log("sources done");
   };
 
 
-  exports.updateSources = updateSources;
+  exports.updateSources = updateSources;

scripts/catalog/utilities.js

@@ -133,11 +133,7 @@ const doesCatalogItemExist = (item) => {
     let content = `---\ntitle: '${item.display_name} Source'\nhidden: true\n---`;
 
     if (!docsPath.includes('/sources/')) {
-      let betaFlag = '';
-      if (item.status === 'PUBLIC_BETA') {
-        betaFlag = 'beta: true\n';
-      }
-      content = `---\ntitle: '${item.display_name} Destination'\nhidden: true\nid: ${item.id}\npublished: false\n${betaFlag}---\n`;
+      content = `---\ntitle: '${item.display_name} Destination'\nhidden: true\nid: ${item.id}\npublished: false\n`;
     }
 
     fs.mkdirSync(docsPath);
@@ -172,4 +168,4 @@ exports.getCatalog = getCatalog;
 exports.getConnectionModes = getConnectionModes;
 exports.isCatalogItemHidden = isCatalogItemHidden;
 exports.sanitize = sanitize;
-exports.doesCatalogItemExist = doesCatalogItemExist;
+exports.doesCatalogItemExist = doesCatalogItemExist;

src/_data/catalog/beta_sources.yml

@@ -1,5 +1,5 @@
 # AUTOGENERATED FROM PUBLIC API. DO NOT EDIT
-# destination categories last updated 2024-05-28 
+# destination categories last updated 2025-01-23 
 items:
 - display_name: A/B Testing
   slug: a-b-testing