11#!/usr/bin/env python
2+ # -*- coding: utf-8 -*-
23#
34# Copyright (c) nexB Inc. and others. All rights reserved.
45# ScanCode is a trademark of nexB Inc.
2425import packageurl
2526import requests
2627import saneyaml
27- import utils_pip_compatibility_tags
2828from commoncode import fileutils
2929from commoncode .hash import multi_checksums
3030from commoncode .text import python_safe_name
3131from packvers import tags as packaging_tags
3232from packvers import version as packaging_version
3333
34+ import utils_pip_compatibility_tags
35+
3436"""
3537Utilities to manage Python thirparty libraries source, binaries and metadata in
3638local directories and remote repositories.
9193
9294- parse requirement file
9395- create a TODO queue of requirements to process
94- - done: create an empty map of processed binary requirements as
95- {package name: (list of versions/tags}
96+ - done: create an empty map of processed binary requirements as {package name: (list of versions/tags}
9697
9798
9899- while we have package reqs in TODO queue, process one requirement:
114115TRACE_ULTRA_DEEP = False
115116
116117# Supported environments
117- PYTHON_VERSIONS = "37 " , "38 " , "39 " , "310 "
118+ PYTHON_VERSIONS = "39 " , "310 " , "311 " , "312" , "313 "
118119
119120PYTHON_DOT_VERSIONS_BY_VER = {
120- "37" : "3.7" ,
121- "38" : "3.8" ,
122121 "39" : "3.9" ,
123122 "310" : "3.10" ,
123+ "311" : "3.11" ,
124+ "312" : "3.12" ,
125+ "313" : "3.13" ,
124126}
125127
126128
@@ -132,10 +134,11 @@ def get_python_dot_version(version):
132134
133135
134136ABIS_BY_PYTHON_VERSION = {
135- "37" : ["cp37" , "cp37m" , "abi3" ],
136- "38" : ["cp38" , "cp38m" , "abi3" ],
137137 "39" : ["cp39" , "cp39m" , "abi3" ],
138138 "310" : ["cp310" , "cp310m" , "abi3" ],
139+ "311" : ["cp311" , "cp311m" , "abi3" ],
140+ "312" : ["cp312" , "cp312m" , "abi3" ],
141+ "313" : ["cp313" , "cp313m" , "abi3" ],
139142}
140143
141144PLATFORMS_BY_OS = {
@@ -553,8 +556,7 @@ def download(self, dest_dir=THIRDPARTY_DIR):
553556 Download this distribution into `dest_dir` directory.
554557 Return the fetched filename.
555558 """
556- if not self .filename :
557- raise ValueError (f"self.filename has no value but is required: { self .filename !r} )
559+ assert self .filename
558560 if TRACE_DEEP :
559561 print (
560562 f"Fetching distribution of { self .name } { self .version } ,
@@ -822,9 +824,9 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False):
822824 """
823825 urls = LinksRepository .from_url (use_cached_index = use_cached_index ).links
824826 errors = []
825- extra_lic_names = [lic .get ("file" ) for lic in self .extra_data .get ("licenses" , {})]
827+ extra_lic_names = [l .get ("file" ) for l in self .extra_data .get ("licenses" , {})]
826828 extra_lic_names += [self .extra_data .get ("license_file" )]
827- extra_lic_names = [eln for eln in extra_lic_names if eln ]
829+ extra_lic_names = [ln for ln in extra_lic_names if ln ]
828830 lic_names = [f"{ key } for key in self .get_license_keys ()]
829831 for filename in lic_names + extra_lic_names :
830832 floc = os .path .join (dest_dir , filename )
@@ -844,7 +846,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False):
844846 if TRACE :
845847 print (f"Fetched license from remote: { lic_url } )
846848
847- except Exception :
849+ except :
848850 try :
849851 # try licensedb second
850852 lic_url = f"{ LICENSEDB_API_URL } { filename }
@@ -857,9 +859,8 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False):
857859 if TRACE :
858860 print (f"Fetched license from licensedb: { lic_url } )
859861
860- except Exception :
861- msg = f"No text for license { filename }
862- f"{ self .license_expression !r} { self }
862+ except :
863+ msg = f'No text for license { filename } { self .license_expression } { self }
863864 print (msg )
864865 errors .append (msg )
865866
@@ -999,7 +1000,7 @@ def get_license_link_for_filename(filename, urls):
9991000 exception if no link is found or if there are more than one link for that
10001001 file name.
10011002 """
1002- path_or_url = [url for url in urls if url .endswith (f"/{ filename } )]
1003+ path_or_url = [l for l in urls if l .endswith (f"/{ filename } )]
10031004 if not path_or_url :
10041005 raise Exception (f"Missing link to file: { filename } )
10051006 if not len (path_or_url ) == 1 :
@@ -1288,7 +1289,7 @@ def is_pure(self):
12881289def is_pure_wheel (filename ):
12891290 try :
12901291 return Wheel .from_filename (filename ).is_pure ()
1291- except Exception :
1292+ except :
12921293 return False
12931294
12941295
@@ -1484,7 +1485,8 @@ def get_distributions(self):
14841485 """
14851486 if self .sdist :
14861487 yield self .sdist
1487- yield from self .wheels
1488+ for wheel in self .wheels :
1489+ yield wheel
14881490
14891491 def get_url_for_filename (self , filename ):
14901492 """
@@ -1613,8 +1615,7 @@ class PypiSimpleRepository:
16131615 type = dict ,
16141616 default = attr .Factory (lambda : defaultdict (dict )),
16151617 metadata = dict (
1616- help = "Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} "
1617- "available in this repo"
1618+ help = "Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} available in this repo"
16181619 ),
16191620 )
16201621
@@ -1628,8 +1629,7 @@ class PypiSimpleRepository:
16281629 type = bool ,
16291630 default = False ,
16301631 metadata = dict (
1631- help = "If True, use any existing on-disk cached PyPI index files. "
1632- "Otherwise, fetch and cache."
1632+ help = "If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache."
16331633 ),
16341634 )
16351635
@@ -1638,8 +1638,7 @@ def _get_package_versions_map(self, name):
16381638 Return a mapping of all available PypiPackage version for this package name.
16391639 The mapping may be empty. It is ordered by version from oldest to newest
16401640 """
1641- if not name :
1642- raise ValueError (f"name is required: { name !r} )
1641+ assert name
16431642 normalized_name = NameVer .normalize_name (name )
16441643 versions = self .packages [normalized_name ]
16451644 if not versions and normalized_name not in self .fetched_package_normalized_names :
@@ -1694,7 +1693,7 @@ def fetch_links(self, normalized_name):
16941693 )
16951694 links = collect_urls (text )
16961695 # TODO: keep sha256
1697- links = [link .partition ("#sha256=" ) for link in links ]
1696+ links = [l .partition ("#sha256=" ) for l in links ]
16981697 links = [url for url , _ , _sha256 in links ]
16991698 return links
17001699
@@ -1915,7 +1914,7 @@ def get_remote_file_content(
19151914 # several redirects and that we can ignore content there. A HEAD request may
19161915 # not get us this last header
19171916 print (f" DOWNLOADING: { url } )
1918- with requests .get (url , allow_redirects = True , stream = True , headers = headers ) as response : # noqa: S113
1917+ with requests .get (url , allow_redirects = True , stream = True , headers = headers ) as response :
19191918 status = response .status_code
19201919 if status != requests .codes .ok : # NOQA
19211920 if status == 429 and _delay < 20 :
@@ -2134,7 +2133,7 @@ def call(args, verbose=TRACE):
21342133 """
21352134 if TRACE_DEEP :
21362135 print ("Calling:" , " " .join (args ))
2137- with subprocess .Popen ( # noqa: S603
2136+ with subprocess .Popen (
21382137 args , stdout = subprocess .PIPE , stderr = subprocess .PIPE , encoding = "utf-8"
21392138 ) as process :
21402139 stdouts = []
@@ -2199,7 +2198,7 @@ def download_wheels_with_pip(
21992198 cli_args .extend (["--requirement" , req_file ])
22002199
22012200 if TRACE :
2202- print ("Downloading wheels using command:" , " " .join (cli_args ))
2201+ print (f "Downloading wheels using command:"" " .join (cli_args ))
22032202
22042203 existing = set (os .listdir (dest_dir ))
22052204 error = False
@@ -2232,7 +2231,7 @@ def download_wheels_with_pip(
22322231
22332232def check_about (dest_dir = THIRDPARTY_DIR ):
22342233 try :
2235- subprocess .check_output (f"venv/bin/about check { dest_dir } .split ()) # noqa: S603
2234+ subprocess .check_output (f"venv/bin/about check { dest_dir } .split ())
22362235 except subprocess .CalledProcessError as cpe :
22372236 print ()
22382237 print ("Invalid ABOUT files:" )
@@ -2283,5 +2282,5 @@ def get_license_expression(declared_licenses):
22832282 return get_only_expression_from_extracted_license (declared_licenses )
22842283 except ImportError :
22852284 # Scancode is not installed, clean and join all the licenses
2286- lics = [python_safe_name (lic ).lower () for lic in declared_licenses ]
2285+ lics = [python_safe_name (l ).lower () for l in declared_licenses ]
22872286 return " AND " .join (lics ).lower ()
0 commit comments