Fix issue #44 (#66)

* add script to process reference docs

* attempt to build with remote cache

* typo

* add reference mdx files

* use html -> mdx pandoc converter

* add reference mdx files

* golang converter from html to md

* remove reference files

* test better converter

* Run converter after upstream docs are copied

* remove yaml files

* fix missing links

* fix more broken links

* cleanup

Author: alan707

.github/workflows/pull-from-bazel-build.yml

@@ -31,22 +31,77 @@ jobs:
     steps:
         - uses: actions/checkout@v5
           with:
-            # Don’t auto-init submodules
+            # Don't auto-init submodules
             submodules: false
 
         - name: Checkout submodules
           run: git submodule update --init -- upstream
-        
+
         - name: Checkout commit of Bazel Build submodule
           if: ${{ inputs.bazelCommitHash != '' }}
           working-directory: upstream
           run: git checkout '${{ inputs.bazelCommitHash }}'
 
+        - name: Setup Bazel
+          uses: bazel-contrib/setup-bazel@0.15.0
+          with:
+            bazelisk-cache: true
+            repository-cache: true
+       
+        - name: Build reference documentation
+          working-directory: upstream
+          run: >
+            bazel build
+            --config=docs
+            --build_metadata=ROLE=DOCS
+            --remote_header=x-buildbuddy-api-key=${{ secrets.BUILDBUDDY_ORG_API_KEY }}
+            --bes_results_url=https://app.buildbuddy.io/invocation/
+            --bes_backend=grpcs://remote.buildbuddy.io
+            --remote_cache=grpcs://remote.buildbuddy.io
+            --remote_timeout=10m
+            //src/main/java/com/google/devtools/build/lib:gen_reference_docs
+
+        # Upload reference-docs.zip as an artifact for debugging purposes
+        - name: Upload reference docs artifact
+          if: ${{ github.ref != 'refs/heads/main' }}
+          uses: actions/upload-artifact@v4.6.2
+          with:
+            name: reference-docs
+            path: upstream/bazel-bin/src/main/java/com/google/devtools/build/lib/reference-docs.zip
+            retention-days: 7
+
         - name: Clean up mdx files
           run: ./cleanup-mdx.sh
 
+        - name: Set up Go
+          uses: actions/setup-go@v6
+          with:
+            go-version: '1.25.2'
+
+        - name: Initialize Go module for converter
+          run: |
+            cd html2md_converter
+            go mod init html-to-md-converter
+            go get github.com/JohannesKaufmann/html-to-markdown
+
+        - name: Build HTML to Markdown converter
+          run: |
+            cd html2md_converter
+            go build -o html-to-md main.go
+            
+        - name: Convert reference documentation HTML to Markdown
+          run: |
+            # Extract and convert HTML reference docs to Markdown
+            ./html2md_converter/html-to-md \
+              -zip upstream/bazel-bin/src/main/java/com/google/devtools/build/lib/reference-docs.zip \
+              -output reference-docs-temp
+
         - name: Transform upstream docs to mdx
           run: ./copy-upstream-docs.sh
 
         - name: Create versioned navigation
           run: ./docs.json.update.sh
+    
+        - name: Clean up temporary files
+          run: rm -rf reference-docs-temp
+

copy-upstream-docs.sh

@@ -6,8 +6,15 @@
 
 set -o errexit -o nounset -o pipefail
 
-SOURCE_DIR="upstream/site/en"
-DEST_DIR="${1:-.}"  # Use first argument or current directory as default
+# Primary upstream directory
+UPSTREAM_SITE="upstream/site/en"
+
+# Reference docs directory
+REFERENCE_DOCS="reference-docs-temp"
+
+# Destination directory (default to current directory)
+DEST_DIR="${1:-.}"
+
 # Files that live in this repo, not fetched from upstream
 LOCAL_FILES="
     index.mdx
@@ -43,6 +50,10 @@ query/language.mdx
 query/quickstart.mdx
 reference/flag-cheatsheet.mdx
 reference/test-encyclopedia.mdx
+reference/command-line-reference.mdx
+reference/be/be-nav.mdx
+reference/be/functions.mdx
+reference/be/platforms-and-toolchains.mdx
 release/rolling.mdx
 remote/ci.mdx
 remote/dynamic.mdx
@@ -53,53 +64,56 @@ start/go.mdx
 tutorials/ccp-toolchain-config.mdx
 "
 
-# Check if source directory exists
-if [ ! -d "$SOURCE_DIR" ]; then
-    echo "Error: Source directory '$SOURCE_DIR' not found"
+# Verify that at least one source exists
+if [ ! -d "$UPSTREAM_SITE" ] && [ ! -d "$REFERENCE_DOCS" ]; then
+    echo "Error: neither source directory exists: '$UPSTREAM_SITE' or '$REFERENCE_DOCS'"
     exit 1
 fi
 
-# Create destination directory if it doesn't exist
 if [ ! -d "$DEST_DIR" ]; then
     echo "Creating destination directory: $DEST_DIR"
     mkdir -p "$DEST_DIR"
 fi
 
-echo "Finding all .md files in $SOURCE_DIR and copying to $DEST_DIR..."
-
-# Find all .md files and copy them
-find "$SOURCE_DIR" -name "*.md" -type f | while read -r source_file; do
-    # Get relative path from upstream/site/en
-    relative_path="${source_file#$SOURCE_DIR/}"
-    
-    # Convert .md to .mdx
-    target_file="${relative_path%.md}.mdx"
-    
-    # Create target directory if it doesn't exist
-    target_dir=$(dirname "$DEST_DIR/$target_file")
-    if [ "$target_dir" != "$DEST_DIR" ]; then
-        mkdir -p "$target_dir"
+echo "Will search in '$UPSTREAM_SITE' and '$REFERENCE_DOCS' (if exists) to copy .md → .mdx to $DEST_DIR"
+
+transform_docs() {
+    local SOURCE_DIR="$1"
+    if [ ! -d "$SOURCE_DIR" ]; then
+        echo "Warning: source directory '$SOURCE_DIR' not found, skipping"
+        return
     fi
-    
+
+    find "$SOURCE_DIR" -name "*.md" -type f | while read -r source_file; do
+        # Derive the relative path inside the source tree
+        relative_path="${source_file#$SOURCE_DIR/}"
+        target_file="${relative_path%.md}.mdx"
+        target_dir=$(dirname "$DEST_DIR/$target_file")
+
+        mkdir -p "$target_dir"
+
     # Check if this file is in the BROKEN_FILES list
-    if echo "$BROKEN_FILES" | grep -q "^$target_file$"; then
-        echo "Skipping broken file: $target_file"
-        continue
-    fi
-    
+        if echo "$BROKEN_FILES" | grep -q "^$target_file$"; then
+            echo "Skipping broken file: $target_file"
+            continue
+        fi
+
     # Transform and copy the file
     echo "Transforming and copying $source_file to $DEST_DIR/$target_file"
     awk -f transform-docs.awk "$source_file" > "$DEST_DIR/$target_file"
-done
+    done
+}
 
-echo "Successfully copied all .md files to .mdx files in $DEST_DIR"
+# Copy from both sources
+transform_docs "$UPSTREAM_SITE"
+transform_docs "$REFERENCE_DOCS"
 
-# Convert community YAML files to MDX
 echo "Converting community YAML files to MDX..."
 ./convert-community-to-mdx.sh "$DEST_DIR/community/experts"
 ./convert-community-to-mdx.sh "$DEST_DIR/community/partners"
 
-# Copy community images to destination community/images/
-# We don't need to do this for images under a docs/ folder, so many other images already work
+echo "Copying community images..."
 mkdir -p "$DEST_DIR/community/images"
 cp upstream/site/en/community/images/* "$DEST_DIR/community/images/"
+
+echo "Done copying docs."