Merge branch 'main' into urlStore_tud

- Resolved conflicts in src/index.js and src/routes/index.js
- Kept both URL Store Controller and PTA2 Controller features

Author: HollywoodTonight

.github/workflows/ci.yaml

@@ -88,6 +88,41 @@ jobs:
           AWS_REGION: us-east-1
           AWS_ACCOUNT_ID: ${{secrets.AWS_ACCOUNT_ID_STAGE}}
 
+  upload-build-artifacts:
+    runs-on: ubuntu-latest
+    needs: build
+    if: github.ref == 'refs/heads/main'
+    steps:
+      - name: Check out
+        uses: actions/checkout@v5
+        with:
+          persist-credentials: 'false'
+
+      - name: Setup Node & NPM
+        uses: ./.github/actions/setup-node-npm
+
+      - name: Configure AWS for DEV
+        uses: ./.github/actions/configure-aws
+        with:
+          aws_role_to_assume: 'arn:aws:iam::${{secrets.AWS_ACCOUNT_ID_DEV}}:role/spacecat-role-github-actions'
+
+      - name: Create Build Artifacts
+        run: npm run build
+        env:
+          AWS_REGION: us-east-1
+          AWS_ACCOUNT_ID: ${{secrets.AWS_ACCOUNT_ID_DEV}}
+
+      - name: Copy Build Artifact
+        run: cp dist/spacecat-services/api-service@*.zip spacecat-services--api-service-latest.zip
+        env:
+          AWS_REGION: us-east-1
+          AWS_ACCOUNT_ID: ${{secrets.AWS_ACCOUNT_ID_DEV}}
+
+      - name: Upload ZIP File
+        run: aws s3 cp spacecat-services--api-service-latest.zip s3://spacecat-artifacts-dev/spacecat-services--api-service-latest.zip
+        env:
+          AWS_REGION: us-east-1
+          AWS_ACCOUNT_ID: ${{secrets.AWS_ACCOUNT_ID_DEV}}
 
   branch-deploy:
     runs-on: ubuntu-latest

.nycrc.json

@@ -16,4 +16,4 @@
     "src/agents/org-detector/instructions.js",
     "src/controllers/demo.js"
   ]
-}
+}

CHANGELOG.md

@@ -7,7 +7,7 @@ info:
     The SpaceCat API is used to manage edge delivery site directory and obtain audit information.
 
     ## Authentication
-    <SecurityDefinitions />
+    <SecurityDefinitions></SecurityDefinitions>
 
   termsOfService: 'https://www.adobe.com/go/experiencecloudterms'
   contact:
@@ -152,6 +152,8 @@ paths:
     $ref: './scrape-api.yaml#/get-scrape-jobs-by-base-url'
   /tools/scrape/jobs/by-base-url/{base64BaseUrl}/by-processingtype/{processingType}:
     $ref: './scrape-api.yaml#/get-scrape-jobs-by-base-url-and-processing-type'
+  /tools/scrape/jobs/by-url/{base64Url}/{processingType}:
+    $ref: './scrape-api.yaml#/get-scrape-urls-by-url-and-processing-type'
   /consent-banner:
     $ref: './consent-banner-api.yaml#/consent-banner'
   /sites:
@@ -190,8 +192,16 @@ paths:
     $ref: './site-opportunities.yaml#/site-opportunity'
   /sites/{siteId}/opportunities/{opportunityId}/suggestions:
     $ref: './site-opportunities.yaml#/site-opportunity-suggestions'
+  /sites/{siteId}/opportunities/{opportunityId}/suggestions/paged/{limit}:
+    $ref: './site-opportunities.yaml#/site-opportunity-suggestions-paged'
+  /sites/{siteId}/opportunities/{opportunityId}/suggestions/paged/{limit}/{cursor}:
+    $ref: './site-opportunities.yaml#/site-opportunity-suggestions-paged-with-cursor'
   /sites/{siteId}/opportunities/{opportunityId}/suggestions/by-status/{status}:
     $ref: './site-opportunities.yaml#/site-opportunity-suggestions-by-status'
+  /sites/{siteId}/opportunities/{opportunityId}/suggestions/by-status/{status}/paged/{limit}:
+    $ref: './site-opportunities.yaml#/site-opportunity-suggestions-by-status-paged'
+  /sites/{siteId}/opportunities/{opportunityId}/suggestions/by-status/{status}/paged/{limit}/{cursor}:
+    $ref: './site-opportunities.yaml#/site-opportunity-suggestions-by-status-paged-with-cursor'
   /sites/{siteId}/opportunities/{opportunityId}/suggestions/{suggestionId}:
     $ref: './site-opportunities.yaml#/site-opportunity-suggestion'
   /sites/{siteId}/opportunities/{opportunityId}/suggestions/status:
@@ -248,6 +258,8 @@ paths:
     $ref: './site-paid.yaml#/site-traffic-paid-type-campaign'
   /sites/{siteId}/traffic/paid/type:
     $ref: './site-paid.yaml#/site-traffic-paid-type'
+  /sites/{siteId}/traffic/paid/pta2/weekly-summary:
+    $ref: './site-paid.yaml#/site-traffic-paid-pta2-weekly-summary'
   /sites/{siteId}/metrics/{metric}/{source}:
     $ref: './site-metrics-api.yaml#/site-metrics-by-source'
   /sites/{siteId}/metrics/{metric}/{source}/by-url/{base64PageUrl}:
@@ -284,6 +296,8 @@ paths:
     $ref: './llmo-api.yaml#/llmo-sheet-data'
   /sites/{siteId}/llmo/sheet-data/{sheetType}/{dataSource}:
     $ref: './llmo-api.yaml#/llmo-sheet-data-with-type'
+  /sites/{siteId}/llmo/sheet-data/{sheetType}/{week}/{dataSource}:
+    $ref: './llmo-api.yaml#/llmo-sheet-data-with-type-and-week'
   /sites/{siteId}/llmo/global-sheet-data/{configName}:
     $ref: './llmo-api.yaml#/llmo-global-sheet-data'
   /sites/{siteId}/llmo/config:

docs/index.html

@@ -145,6 +145,94 @@ llmo-sheet-data-with-type:
     security:
       - api_key: [ ]
 
+llmo-sheet-data-with-type-and-week:
+  parameters:
+    - $ref: './parameters.yaml#/siteId'
+    - name: sheetType
+      in: path
+      required: true
+      description: The sheet type identifier to fetch from the external endpoint
+      schema:
+        type: string
+        example: 'analytics'
+    - name: week
+      in: path
+      required: true
+      description: The week identifier (e.g., w01, w02) to fetch time-series data
+      schema:
+        type: string
+        pattern: '^w\d{2}$'
+        example: 'w01'
+    - name: dataSource
+      in: path
+      required: true
+      description: The data source identifier to fetch from the external endpoint
+      schema:
+        type: string
+        example: 'questions'
+  get:
+    tags:
+      - llmo
+    summary: Get LLMO sheet data with sheet type and week
+    description: |
+      Retrieves data from the external LLMO data endpoint for a specific site with a specified sheet type and week.
+      This endpoint proxies data from the external HLX API based on the site's LLMO configuration.
+      The data is fetched from the path: {dataFolder}/{sheetType}/{week}/{dataSource}.json
+      This is useful for fetching time-series data organized by week.
+    operationId: getLlmoSheetDataWithTypeAndWeek
+    responses:
+      '200':
+        description: LLMO sheet data retrieved successfully
+        content:
+          application/json:
+            schema:
+              type: object
+              description: The data returned from the external LLMO endpoint
+              additionalProperties: true
+      '400':
+        $ref: './responses.yaml#/400'
+      '401':
+        $ref: './responses.yaml#/401'
+      '500':
+        $ref: './responses.yaml#/500'
+    security:
+      - api_key: [ ]
+  post:
+    tags:
+      - llmo
+    summary: Query LLMO sheet data with sheet type and week using filters, exclusions, and grouping
+    description: |
+      Retrieves and processes data from the external LLMO data endpoint for a specific site with a specified sheet type,
+      week, and advanced querying capabilities. This endpoint allows filtering data with case-insensitive exact matching,
+      excluding specific fields from the response, and grouping data by specified attributes.
+      The data is fetched from the path: {dataFolder}/{sheetType}/{week}/{dataSource}.json
+      The endpoint fetches all available data (up to 1M records) to apply the query operations effectively.
+      This is useful for fetching and analyzing time-series data organized by week.
+    operationId: queryLlmoSheetDataWithTypeAndWeek
+    requestBody:
+      required: false
+      content:
+        application/json:
+          schema:
+            $ref: './schemas.yaml#/LlmoSheetDataQuery'
+    responses:
+      '200':
+        description: LLMO sheet data queried and processed successfully
+        content:
+          application/json:
+            schema:
+              type: object
+              description: The processed data returned from the external LLMO endpoint after applying filters, exclusions, and grouping
+              additionalProperties: true
+      '400':
+        $ref: './responses.yaml#/400'
+      '401':
+        $ref: './responses.yaml#/401'
+      '500':
+        $ref: './responses.yaml#/500'
+    security:
+      - api_key: [ ]
+
 llmo-global-sheet-data:
   parameters:
     - $ref: './parameters.yaml#/siteId'
@@ -194,6 +282,7 @@ llmo-config:
       Retrieves the LLMO (Large Language Model Optimizer) configuration for a specific site.
       This endpoint retrieves the configuration directly from S3 storage. The configuration
       includes entities (categories, topics), brand aliases, and competitor information.
+      Categories may include optional URLs for content matching.
 
       If a version parameter is provided, the specific version of the configuration will be
       retrieved. If the version does not exist, a 404 error will be returned. If no version
@@ -238,6 +327,9 @@ llmo-config:
       This operation completely overwrites the existing configuration with the provided payload.
       The payload is validated against the LLMO configuration schema before being stored in S3.
       The configuration must include entities, brands, and competitors sections.
+      
+      Categories can optionally include URLs that can be either exact URLs or URL prefixes
+      (ending with /*) to match against specific content areas.
     operationId: postLlmoConfig
     requestBody:
       required: true

docs/openapi/api.yaml

@@ -49,6 +49,14 @@ base64PageUrl:
   schema:
     type: string
     format: base64url
+base64Url:
+  name: base64Url
+  description: Base64-encoded URL
+  in: path
+  required: true
+  schema:
+    type: string
+    format: base64url
 ascending:
   name: ascending
   description: Whether to sort ascending or descending
@@ -216,6 +224,13 @@ processingType:
   required: true
   schema:
     type: string
+processingTypeOptional:
+  name: processingType
+  description: Optional, The processing type/handler used by the scrape job, will use "default" if not provided
+  in: path
+  required: false
+  schema:
+    type: string
 weekNumber:
   name: weekNumber
   description: The ISO week number (1-53)
@@ -317,7 +332,7 @@ externalId:
 sandboxAuditType:
   name: auditType
   description: |
-    Comma-separated list of audit types to run on sandbox sites. 
+    Comma-separated list of audit types to run on sandbox sites.
     Use 'product-metatags' for specialized Product Detail Page (PDP) analysis of commerce sites.
   in: query
   required: false
@@ -343,3 +358,24 @@ llmoConfigVersion:
     type: string
     description: S3 object version ID
     example: "abc123def456"
+
+limit:
+  name: limit
+  description: The maximum number of items to return per page
+  in: path
+  required: true
+  schema:
+    type: integer
+    minimum: 1
+    maximum: 1000
+    default: 100
+    example: 100
+
+cursor:
+  name: cursor
+  description: The cursor for pagination to fetch the next page of results
+  in: path
+  required: false
+  schema:
+    type: string
+    example: "eyJzayI6IjIwMjQtMDEtMTVUMTI6MzA6MDBaIiwiaWQiOiIxMjNlNDU2Ny1lODliLTEyZDMtYTQ1Ni00MjY2MTQxNzQwMDAifQ=="