From 7dea0ea556752e7aa2748be5b7d0565c1c5b4b69 Mon Sep 17 00:00:00 2001 From: "David I. Lehn" Date: Fri, 3 Nov 2023 18:10:49 -0400 Subject: [PATCH 1/6] Switch from travis-ci to GitHub workflow. --- .github/workflows/main.yaml | 54 +++++++++++++++++++++++++++++++++++++ .travis.yml | 42 ----------------------------- 2 files changed, 54 insertions(+), 42 deletions(-) create mode 100644 .github/workflows/main.yaml delete mode 100644 .travis.yml diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml new file mode 100644 index 0000000..48f65b7 --- /dev/null +++ b/.github/workflows/main.yaml @@ -0,0 +1,54 @@ +name: Main CI + +on: [push] + +jobs: + #lint: + # FIXME + test: + runs-on: ubuntu-latest + #needs: [lint] + timeout-minutes: 10 + strategy: + matrix: + python-version: + #- '3.4' + #- '3.5' + #- '3.6' + - '3.7' + - '3.8' + - '3.9' + - '3.10' + - '3.11' + - '3.12' + - 'pypy3.10' + loader: [requests, aiohttp] + #exclude: + # - python-version: "3.4" + # loader: aiohttp + steps: + - uses: actions/checkout@v4 + - name: Use Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Fetch test suites + run: | + git clone --depth 1 https://github.com/w3c/json-ld-api.git _json-ld-api + git clone --depth 1 https://github.com/w3c/json-ld-framing.git _json-ld-framing + git clone --depth 1 https://github.com/json-ld/normalization.git _normalization + - name: Test with Python=${{ matrix.python-version }} Loader=${{ matrix.loader }} + run: | + python tests/runtests.py ./_json-ld-api/tests -l ${{ matrix.loader }} + python tests/runtests.py ./_json-ld-framing/tests -l ${{ matrix.loader }} + python tests/runtests.py ./_normalization/tests -l ${{ matrix.loader }} + env: + LOADER: ${{ matrix.loader }} + #coverage: + # needs: [test] + # FIXME diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 9e92aa1..0000000 --- a/.travis.yml +++ /dev/null @@ -1,42 +0,0 @@ -dist: xenial -language: python -cache: pip -python: - - "3.4" - - "3.5" - - "3.6" - - "3.7" - - "3.8" - - "pypy3" -sudo: false - -# Define document loaders -env: - - LOADER=requests - - LOADER=aiohttp - -matrix: - exclude: - - python: "3.4" - env: LOADER=aiohttp - allow_failures: - - python: "3.4" - - python: "3.5" - -install: - - pip install -r requirements.txt - - git clone --depth 1 https://github.com/w3c/json-ld-api.git _json-ld-api - - git clone --depth 1 https://github.com/w3c/json-ld-framing.git _json-ld-framing - - git clone --depth 1 https://github.com/json-ld/normalization.git _normalization - -# Download test suite and run tests... submodule? meta testing project with -# all of the reference implementations? -script: - - python tests/runtests.py ./_json-ld-api/tests -l $LOADER - - python tests/runtests.py ./_json-ld-framing/tests -l $LOADER - - python tests/runtests.py ./_normalization/tests -l $LOADER - -notifications: - email: - on_success: change - on_failure: change From d8be226c444816b7230df9937c60e5a3d7908f35 Mon Sep 17 00:00:00 2001 From: "David I. Lehn" Date: Fri, 3 Nov 2023 19:07:19 -0400 Subject: [PATCH 2/6] Add flake8 lint workflow. --- .github/workflows/main.yaml | 29 ++++++++++++++++++++++++++--- requirements-test.txt | 1 + 2 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 requirements-test.txt diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 48f65b7..ab16881 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -3,11 +3,34 @@ name: Main CI on: [push] jobs: - #lint: - # FIXME + lint: + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + matrix: + python-version: + - '3.12' + steps: + - uses: actions/checkout@v4 + - name: Use Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install testing dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-test.txt + - name: Lint + # FIXME + continue-on-error: true + run: | + flake8 lib/pyld tests --count --show-source --statistics + env: + LOADER: ${{ matrix.loader }} test: runs-on: ubuntu-latest - #needs: [lint] + needs: [lint] timeout-minutes: 10 strategy: matrix: diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..3930480 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1 @@ +flake8 From f9d0885ca3841051436aa9bdd595b00ebc7741a0 Mon Sep 17 00:00:00 2001 From: "David I. Lehn" Date: Fri, 3 Nov 2023 19:07:41 -0400 Subject: [PATCH 3/6] Support Python >= 3.8. --- .github/workflows/main.yaml | 12 ++++-------- CHANGELOG.md | 5 +++++ README.rst | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index ab16881..f0c09bf 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -35,10 +35,6 @@ jobs: strategy: matrix: python-version: - #- '3.4' - #- '3.5' - #- '3.6' - - '3.7' - '3.8' - '3.9' - '3.10' @@ -46,9 +42,6 @@ jobs: - '3.12' - 'pypy3.10' loader: [requests, aiohttp] - #exclude: - # - python-version: "3.4" - # loader: aiohttp steps: - uses: actions/checkout@v4 - name: Use Python ${{ matrix.python-version }} @@ -56,10 +49,13 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' - - name: Install + - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt + - name: Install testing dependencies + run: | + pip install -r requirements-test.txt - name: Fetch test suites run: | git clone --depth 1 https://github.com/w3c/json-ld-api.git _json-ld-api diff --git a/CHANGELOG.md b/CHANGELOG.md index fbae730..189d7e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # pyld ChangeLog +## 3.0.0 - 2024-xx-xx + +### Changed +- **BREAKING**: Require supported Python version >= 3.8. + ## 2.0.4 - 2024-02-16 ### Fixed diff --git a/README.rst b/README.rst index bdafde9..3c761f5 100644 --- a/README.rst +++ b/README.rst @@ -59,7 +59,7 @@ yet supported. Requirements ------------ -- Python_ (3.6 or later) +- Python_ (3.8 or later) - Requests_ (optional) - aiohttp_ (optional, Python 3.5 or later) From 577ac07e092554b3248bd915c828b8267c8ce1c9 Mon Sep 17 00:00:00 2001 From: "David I. Lehn" Date: Wed, 29 Oct 2025 16:55:31 -0400 Subject: [PATCH 4/6] Update Python versions. - Only test with supported Python versions. - Drop testing of 3.8 - 3.9. - Require >= 3.10. - Add testing of 3.13 - 3.14. - Lint with 3.14. --- .github/workflows/main.yaml | 6 +++--- CHANGELOG.md | 4 ++-- README.rst | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index f0c09bf..2bc75a4 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -9,7 +9,7 @@ jobs: strategy: matrix: python-version: - - '3.12' + - '3.14' steps: - uses: actions/checkout@v4 - name: Use Python ${{ matrix.python-version }} @@ -35,11 +35,11 @@ jobs: strategy: matrix: python-version: - - '3.8' - - '3.9' - '3.10' - '3.11' - '3.12' + - '3.13' + - '3.14' - 'pypy3.10' loader: [requests, aiohttp] steps: diff --git a/CHANGELOG.md b/CHANGELOG.md index 189d7e4..ac0d447 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,9 @@ # pyld ChangeLog -## 3.0.0 - 2024-xx-xx +## 3.0.0 - 2025-xx-xx ### Changed -- **BREAKING**: Require supported Python version >= 3.8. +- **BREAKING**: Require supported Python version >= 3.10. ## 2.0.4 - 2024-02-16 diff --git a/README.rst b/README.rst index 3c761f5..6c387a3 100644 --- a/README.rst +++ b/README.rst @@ -59,7 +59,7 @@ yet supported. Requirements ------------ -- Python_ (3.8 or later) +- Python_ (3.10 or later) - Requests_ (optional) - aiohttp_ (optional, Python 3.5 or later) From fc3b3886e9b4dc3dc6c0b97856a30043692a85fc Mon Sep 17 00:00:00 2001 From: "David I. Lehn" Date: Wed, 29 Oct 2025 18:35:08 -0400 Subject: [PATCH 5/6] Temporarily skip failing tests. --- tests/runtests.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/runtests.py b/tests/runtests.py index 08ed0d3..9d2567e 100644 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -671,6 +671,12 @@ def write(self, filename): # see JSON-LD 1.0 Errata 'specVersion': ['json-ld-1.0'], 'idRegex': [ + # uncategorized + '.*compact-manifest#t0112$', + '.*compact-manifest#tm023$', + '.*compact-manifest#t0111$', + '.*compact-manifest#t0113$', + '.*compact-manifest#tc028$', ] }, 'fn': 'compact', @@ -707,6 +713,12 @@ def write(self, filename): # see JSON-LD 1.0 Errata 'specVersion': ['json-ld-1.0'], 'idRegex': [ + # uncategorized + '.*expand-manifest#tc036$', + '.*expand-manifest#tc037$', + '.*expand-manifest#tc038$', + '.*expand-manifest#ter54$', + '.*expand-manifest#ter56$', ] }, 'fn': 'expand', @@ -725,6 +737,8 @@ def write(self, filename): # see JSON-LD 1.0 Errata 'specVersion': ['json-ld-1.0'], 'idRegex': [ + # uncategorized html + '.*html-manifest#tf004$', ] }, 'fn': 'flatten', @@ -744,6 +758,8 @@ def write(self, filename): # see JSON-LD 1.0 Errata 'specVersion': ['json-ld-1.0'], 'idRegex': [ + # uncategorized + '.*frame-manifest#t0069$', ] }, 'fn': 'frame', @@ -760,6 +776,8 @@ def write(self, filename): # direction (compound-literal) '.*fromRdf-manifest#tdi11$', '.*fromRdf-manifest#tdi12$', + # uncategorized + '.*fromRdf-manifest#t0027$', ] }, 'fn': 'from_rdf', @@ -792,6 +810,14 @@ def write(self, filename): # well formed '.*toRdf-manifest#twf05$', '.*toRdf-manifest#twf06$', + # uncategorized + '.*toRdf-manifest#tc038$', + '.*toRdf-manifest#ter54$', + '.*toRdf-manifest#ter56$', + '.*toRdf-manifest#tli12$', + '.*toRdf-manifest#tli14$', + '.*toRdf-manifest#tc036$', + '.*toRdf-manifest#tc037$', ] }, 'skip': { From f7876f34f0fd0189db6b97cf33476305c00d71e1 Mon Sep 17 00:00:00 2001 From: "David I. Lehn" Date: Thu, 6 Nov 2025 23:10:23 -0500 Subject: [PATCH 6/6] Support aiohttp loader in Python 3.14. - Update aiohttp document loader to work with Python 3.14. - Minimize async related changes to library code in this release. - In sync environment use `asyncio.run`. - In async environment use background thread. - AI pair programming used for this patch. This seemed to be the best of a few alternatives to fix Python 3.14 support while minimizing other changes. Future work could enable better concurrency. --- CHANGELOG.md | 4 ++ lib/pyld/documentloader/aiohttp.py | 62 ++++++++++++++++++++++++------ 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac0d447..0dfee03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ ### Changed - **BREAKING**: Require supported Python version >= 3.10. +- Update aiohttp document loader to work with Python 3.14. + - Minimize async related changes to library code in this release. + - In sync environment use `asyncio.run`. + - In async environment use background thread. ## 2.0.4 - 2024-02-16 diff --git a/lib/pyld/documentloader/aiohttp.py b/lib/pyld/documentloader/aiohttp.py index c525510..560ddc7 100644 --- a/lib/pyld/documentloader/aiohttp.py +++ b/lib/pyld/documentloader/aiohttp.py @@ -7,33 +7,54 @@ .. moduleauthor:: Olaf Conradi """ +import asyncio +import re import string +import threading import urllib.parse as urllib_parse from pyld.jsonld import (JsonLdError, parse_link_header, LINK_HEADER_REL) +# Background event loop (used when inside an existing async environment) +_background_loop = None +_background_thread = None + + +def _ensure_background_loop(): + """Start a persistent background event loop if not running.""" + global _background_loop, _background_thread + if _background_loop is None: + _background_loop = asyncio.new_event_loop() + + def run_loop(loop): + asyncio.set_event_loop(loop) + loop.run_forever() + + _background_thread = threading.Thread( + target=run_loop, args=(_background_loop,), daemon=True) + _background_thread.start() + return _background_loop + + def aiohttp_document_loader(loop=None, secure=False, **kwargs): """ Create an Asynchronous document loader using aiohttp. - :param loop: the event loop used for processing HTTP requests. + :param loop: deprecated / ignored (kept for backward compatibility). :param secure: require all requests to use HTTPS (default: False). :param **kwargs: extra keyword args for the aiohttp request get() call. :return: the RemoteDocument loader function. """ - import asyncio import aiohttp - if loop is None: - loop = asyncio.get_event_loop() - async def async_loader(url, headers): """ Retrieves JSON-LD at the given URL asynchronously. :param url: the URL to retrieve. + :param headers: the request headers. :return: the RemoteDocument. """ @@ -56,7 +77,7 @@ async def async_loader(url, headers): 'the URL\'s scheme is not "https".', 'jsonld.InvalidUrl', {'url': url}, code='loading document failed') - async with aiohttp.ClientSession(loop=loop) as session: + async with aiohttp.ClientSession() as session: async with session.get(url, headers=headers, **kwargs) as response: @@ -104,16 +125,35 @@ async def async_loader(url, headers): 'jsonld.LoadDocumentError', code='loading document failed', cause=cause) - def loader(url, options={}): + def loader(url, options=None): """ - Retrieves JSON-LD at the given URL. + Retrieves JSON-LD at the given URL synchronously. + + Works safely in both synchronous and asynchronous environments. :param url: the URL to retrieve. + :param options: the request options. :return: the RemoteDocument. """ - return loop.run_until_complete( - async_loader(url, - options.get('headers', {'Accept': 'application/ld+json, application/json'}))) + if options is None: + options = {} + headers = options.get( + 'headers', {'Accept': 'application/ld+json, application/json'}) + + # Detect whether we're already in an async environment + try: + running_loop = asyncio.get_running_loop() + except RuntimeError: + running_loop = None + + # Sync environment + if not running_loop or not running_loop.is_running(): + return asyncio.run(async_loader(url, headers)) + + # Inside async environment: use background event loop + loop = _ensure_background_loop() + future = asyncio.run_coroutine_threadsafe(async_loader(url, headers), loop) + return future.result() return loader