Skip to content

Commit 9530593

Browse files
authored
Enhance Meilisearch scraper workflow
Added validation for JSON configuration and updated the volume path for the Docker run command.
1 parent bab1efc commit 9530593

File tree

1 file changed

+15
-11
lines changed

1 file changed

+15
-11
lines changed

.github/workflows/meilisearch-scraper.yml

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,13 @@ on:
88
types: [closed]
99
branches:
1010
- main
11-
workflow_dispatch: # Manual trigger option
11+
workflow_dispatch:
1212
schedule:
13-
- cron: '0 2 * * *' # Daily at 2 AM UTC
13+
- cron: '0 2 * * *'
1414

1515
jobs:
1616
scrape-docs:
1717
runs-on: ubuntu-latest
18-
# Only run on main branch or merged PRs
1918
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || (github.event_name == 'pull_request' && github.event.pull_request.merged == true)
2019

2120
steps:
@@ -24,20 +23,25 @@ jobs:
2423

2524
- name: Wait for deployment (if push event)
2625
if: github.event_name == 'push'
27-
run: sleep 60 # Wait for Vercel/Netlify deployment
26+
run: sleep 60
27+
28+
- name: Validate JSON config
29+
run: |
30+
echo "Validating JSON configuration..."
31+
python3 -c "import json; json.load(open('.github/workflows/meilisearch-scraper-config.json'))"
32+
echo "JSON is valid!"
2833
2934
- name: Run Meilisearch Scraper
3035
env:
3136
MEILISEARCH_HOST_URL: ${{ secrets.MEILISEARCH_HOST_URL }}
3237
MEILISEARCH_API_KEY: ${{ secrets.MEILISEARCH_MASTER_KEY }}
33-
CONFIG_FILE: .github/meilisearch/scraper-config.json
3438
run: |
35-
docker run \
36-
-e MEILISEARCH_HOST_URL="$MEILISEARCH_HOST_URL" \
37-
-e MEILISEARCH_API_KEY="$MEILISEARCH_API_KEY" \
38-
-v ${{ github.workspace }}/.github:/config \
39-
getmeili/docs-scraper:v0.12.8 \
40-
pipenv run ./docs_scraper /config/meilisearch-scraper-config.json
39+
docker run \
40+
-e MEILISEARCH_HOST_URL="$MEILISEARCH_HOST_URL" \
41+
-e MEILISEARCH_API_KEY="$MEILISEARCH_API_KEY" \
42+
-v ${{ github.workspace }}/.github/workflows:/config \
43+
getmeili/docs-scraper:v0.12.8 \
44+
pipenv run ./docs_scraper /config/meilisearch-scraper-config.json
4145
4246
- name: Test search index
4347
env:

0 commit comments

Comments
 (0)