bigint pre-release test

Author: pmp-p

config

@@ -20,7 +20,7 @@ export PYTHONPYCACHEPREFIX=${PYTHONPYCACHEPREFIX:-$(realpath ${SDKROOT}/build/py
 export HOME=${SDKROOT}
 
 export HPY=${HOST_PREFIX}/bin/python${PYBUILD}
-export PIP=${HOST_PREFIX}/bin/pip${PYBUILD}
+export HPIP=${HOST_PREFIX}/bin/pip${PYBUILD}
 
 # cpython build opts
 export CPOPTS="-Os -g0 -fPIC"
@@ -49,14 +49,20 @@ fi
 
 EMFLAVOUR=${EMFLAVOUR:-tot}
 
-# no extra wasm features
-export WASMFLAVOUR=mvp
+# base wasm features  pure is "mvp" 
+export WASM_FLAVOUR=bi
+
+# extra features. Safari 14 is expected to have BIGINT, ignore the emsdk warnings.
+export WASM_EXTRA="-sWASM_BIGINT -sPOLYFILL -sMIN_SAFARI_VERSION=140000"
+
+# tell to not normalize modules.
+export WASM_PURE=false
 
 #temp fix for oom on CI (Error: Process completed with exit code 143.)
 export EMSDK_NUM_CORES=1
 
 
-export PYDK_PYTHON_HOST_PLATFORM=wasm32-$WASMFLAVOUR-emscripten
+export PYDK_PYTHON_HOST_PLATFORM=wasm32-${WASM_FLAVOUR}-emscripten
 
 if echo $LD_LIBRARY_PATH |grep -q ${HOST_PREFIX}/lib
 then

scripts/cpython-build-emsdk-prebuilt.sh

@@ -2,21 +2,20 @@
 
 . ${CONFIG:-config}
 
-CYTHON_REL=${CYTHON_REL:-3.0.0}
+CYTHON_REL=${CYTHON_REL:-3.0.1}
 CYTHON_WHL=${CYTHON:-Cython-${CYTHON_REL}-py2.py3-none-any.whl}
 
+# all needed for PEP722/723
 PACKAGING="pip build wheel pyparsing packaging installer"
 
+$HPIP install --upgrade $PACKAGING
 
-$HPY -m pip install --upgrade $PACKAGING
+# setuptools for HPy/static
+$HPIP install --upgrade setuptools
 
-# only for the simulator
-$HPY -m pip install --upgrade aioconsole
+# aioconsole only for the simulator
+$HPIP install --upgrade aioconsole
 
-pushd build
-    wget -q -c https://github.com/cython/cython/releases/download/${CYTHON_REL}/${CYTHON_WHL}
-    $HPY install --upgrade $CYTHON_WHL
-popd
 
 
 PIP="${SDKROOT}/python3-wasm -m pip"
@@ -28,31 +27,47 @@ echo "
 
 # support package build/install
 $HPY -m pip install --upgrade $PACKAGING
+
 $PIP install --upgrade $PACKAGING
 
+# setuptools for HPy/static
+$PIP install --upgrade setuptools
+
+
+
 # make wheels
 # /opt/python-wasm-sdk/python3-wasm setup.py bdist_wheel
 
 
-# cython
-$HPY -m pip install build/$CYTHON_WHL
+# cython get the latest release on gh
+pushd build
+    wget -q -c https://github.com/cython/cython/releases/download/${CYTHON_REL}/${CYTHON_WHL}
+    $HPIP install --upgrade $CYTHON_WHL
+popd
+
 $PIP install build/$CYTHON_WHL
 
 
 # some we want to be certain to have in all minimal rootfs
 mkdir -p prebuilt/emsdk/common/site-packages/
-for pkg in pyparsing packaging installer
+for pkg in pyparsing packaging installer pkg_resources
 do
     if [ -d prebuilt/emsdk/${PYBUILD}/site-packages/$pkg ]
     then
         echo "
             $pkg already set to prebuilt
             "
     else
-        $PIP install $pkg
-        cp -rf $PREFIX/lib/python${PYBUILD}/site-packages/${pkg} prebuilt/emsdk/common/site-packages/
-        # skip the distinfo to save space
-        #cp -rf $PREFIX/lib/python${PYBUILD}/site-packages/${pkg}-* prebuilt/emsdk/common/site-packages/
+        if [ -d ${HOST_PREFIX}/lib/python${PYBUILD}/site-packages/${pkg} ]
+        then
+            cp -rf ${HOST_PREFIX}/lib/python${PYBUILD}/site-packages/${pkg} prebuilt/emsdk/common/site-packages/
+            # skip the distinfo to save some space
+            #cp -rf ${HOST_PREFIX}/lib/python${PYBUILD}/site-packages/${pkg}-* prebuilt/emsdk/common/site-packages/
+
+        else
+            cp -rf ${ROOT}/.local/lib/python${PYBUILD}/site-packages/{$pkg} prebuilt/emsdk/common/site-packages/
+            # skip the distinfo to save some space
+        fi
     fi
 done
 

scripts/cpython-build-emsdk.sh

@@ -165,14 +165,15 @@ END
 
 #     --with-libs='-lz -lffi' \
 
-
-    # MODULE__HASHLIB_LDFLAGS=-L/usr/lib   -lcrypto
     pushd $ROOT/src/cpython${PYBUILD}
+    # fix double linking
     sed -i 's|   -lcrypto||g' Makefile.pre.in
+    # please let compiler/user decide what to do with wasm CPU.
     sed -i 's|-sWASM_BIGINT||g' configure
     sed -i 's|-sWASM_BIGINT||g' configure.ac
     popd
 
+
     PKG_CONFIG_PATH="${PREFIX}/lib/pkgconfig" CONFIG_SITE=$ROOT/src/cpython${PYBUILD}/Tools/wasm/config.site-wasm32-pydk \
     emconfigure $ROOT/src/cpython${PYBUILD}/configure -C --with-emscripten-target=browser \
      --cache-file=${PYTHONPYCACHEPREFIX}/config.cache \
@@ -187,31 +188,6 @@ END
 
     #echo "#define HAVE_NCURSES_H" >> pyconfig.h
 
-    if echo $PYBUILD|grep -q 3.10
-    then
-        cat > Modules/Setup.local <<END
-*disabled*
-_decimal
-xxsubtype
-_crypt
-curses
-
-*static*
-zlib zlibmodule.c
-END
-
-    else
-        cat > Modules/Setup.local <<END
-*static*
-_ctypes _ctypes/_ctypes.c _ctypes/callbacks.c _ctypes/callproc.c _ctypes/stgdict.c _ctypes/cfield.c ${SDKROOT}/emsdk/upstream/emscripten/cache/sysroot/lib/wasm32-emscripten/pic/libffi.a
-
-*disabled*
-_decimal
-xxsubtype
-_crypt
-
-END
-    fi
 
     if emmake make -j$NPROC WASM_ASSETS_DIR=$(realpath ${PYTHONPYCACHEPREFIX}/empty)@/
     then
@@ -282,9 +258,6 @@ sed -i 's|-g0|-g3|g' ${MODSYSCONFIG}
 # this one is required for `python3-wasm -mbuild` venv
 ln ${MODSYSCONFIG} ${SDKROOT}/devices/$(arch)/usr/lib/python${PYBUILD}/
 
-# cmake usually wants cc
-ln ${SDKROOT}/emsdk/upstream/emscripten/emcc ${SDKROOT}/emsdk/upstream/emscripten/cc
-ln ${SDKROOT}/emsdk/upstream/emscripten/emcc.py ${SDKROOT}/emsdk/upstream/emscripten/cc.py
 
 cat > ${PYTHONPYCACHEPREFIX}/.nanorc <<END
 set tabsize 4

scripts/cpython-build-host-deps.sh

@@ -3,13 +3,13 @@
 . ${CONFIG:-config}
 
 echo "
-    *   cpython-build-host-prebuilt pip==$PIP   *
+    *   cpython-build-host-prebuilt pip==$HPIP   *
 " 1>&2
 
 
 # install and update critical packages.
-$PIP install --upgrade pip
+$HPIP install --upgrade pip
 
 
 # 3.12 and git deprecated setuptools bundling.
-$PIP install --upgrade setuptools
+$HPIP install --upgrade setuptools

scripts/cpython-fetch.sh

@@ -17,6 +17,7 @@ PYPATCH=true
 
 [ -f $HPY ] || REBUILD=true
 
+
 if echo $PYBUILD |grep -q 13$
 then
     if [ -d cpython${PYBUILD} ]
@@ -41,6 +42,7 @@ then
     fi
 fi
 
+
 if echo $PYBUILD |grep -q 12$
 then
     if [ -d cpython${PYBUILD} ]
@@ -73,17 +75,6 @@ then
     export REBUILD=true
 fi
 
-if echo $PYBUILD | grep -q 10$
-then
-    wget -q -c https://www.python.org/ftp/python/3.10.6/Python-3.10.6.tar.xz
-    tar xf Python-3.10.6.tar.xz
-
-    ln -s Python-3.10.6 cpython${PYBUILD}
-
-    NOPATCH=true
-    export REBUILD=true
-fi
-
 popd
 
 
@@ -102,7 +93,7 @@ fi
 
 if $NOPATCH
 then
-    echo -n
+    echo "finally there"
 else
     # do some patching for 3.11+ to allow more shared libs
     pushd src/cpython${PYBUILD} 2>&1 >/dev/null

scripts/emsdk-fetch.sh

@@ -173,16 +173,22 @@ SHARED=""
 IS_SHARED=false
 PY_MODULE=false
 MVP=\${MVP:true}
+WASM_PURE=\${WASM_PURE:true}
 
 if \$MVP
 then
     # -mcpu=generic would activate those https://reviews.llvm.org/D125728
     # https://github.com/emscripten-core/emscripten/pull/17689
-    CPU="-sWASM_BIGINT=0 -sMIN_SAFARI_VERSION=120000 -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-mutable-globals -m32"
+    # CPU="-sWASM_BIGINT=0 -sMIN_SAFARI_VERSION=140000 -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-mutable-globals -m32"
+    # go hybrid
+    CPU="-sMIN_SAFARI_VERSION=140000 -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-mutable-globals -m32"
 else
     CPU="-mcpu=bleeding-edge -m32"
 fi
 
+# quick hack until 3.1.47
+CPU="$WASM_EXTRA \$CPU"
+
 
 LINKING=\${LINKING:-false}
 
@@ -243,6 +249,16 @@ for arg do
         continue
     fi
 
+    if [ "\$arg" = "-c" ]
+    then
+        CPU_EXTRA=\$WASM_EXTRA
+    fi
+
+    if [ "\$arg" = "-o" ]
+    then
+        CPU_EXTRA=\$WASM_EXTRA
+    fi
+
     if [ "\$arg" = "-fallow-argument-mismatch" ]
     then
         continue
@@ -320,17 +336,22 @@ done
 
 if \$IS_SHARED
 then
-    $EMSDK_PYTHON -E \$0.py \$SHARED $CPU  $COPTS $LDFLAGS -sSIDE_MODULE -gsource-map --source-map-base / "\$@" \$COMMON
+    # always pass CPU opts when linking
+    $EMSDK_PYTHON -E \$0.py \$SHARED \$CPU $COPTS $LDFLAGS -sSIDE_MODULE -gsource-map --source-map-base / "\$@" \$COMMON
     if \$MVP
     then
-        SOTMP=\$(mktemp).so
-        mv \$SHARED_TARGET \$SOTMP
-        $SDKROOT/emsdk/upstream/bin/wasm-emscripten-finalize -mvp \$SOTMP -o \$SHARED_TARGET
-        [ -f \$SHARED_TARGET.map ] && rm \$SHARED_TARGET.map
-        rm \$SOTMP
+        if \$WASM_PURE
+        then
+            SOTMP=\$(mktemp).so
+            mv \$SHARED_TARGET \$SOTMP
+            $SDKROOT/emsdk/upstream/bin/wasm-emscripten-finalize -mvp \$SOTMP -o \$SHARED_TARGET
+            [ -f \$SHARED_TARGET.map ] && rm \$SHARED_TARGET.map
+            rm \$SOTMP
+        fi
     fi
 else
-    $EMSDK_PYTHON -E \$0.py $CPU -fpic \$COPTS \$CPPFLAGS -DBUILD_STATIC "\$@" \$COMMON
+    # pass CPU opts only when -c/-o but always PIC
+    $EMSDK_PYTHON -E \$0.py \$CPU_EXTRA -fpic \$COPTS \$CPPFLAGS -DBUILD_STATIC "\$@" \$COMMON
 fi
 #else
 #  unset _EMCC_CCACHE
@@ -339,7 +360,23 @@ fi
 
 END
 
-        cat emsdk/upstream/emscripten/emcc > emsdk/upstream/emscripten/em++
+        rm emsdk/upstream/emscripten/em++
+        if ln emsdk/upstream/emscripten/emcc emsdk/upstream/emscripten/em++
+        then
+            # cmake usually wants cc
+            ln emsdk/upstream/emscripten/emcc emsdk/upstream/emscripten/cc
+            ln emsdk/upstream/emscripten/emcc.py emsdk/upstream/emscripten/cc.py
+        else
+            echo "
+
+             ============ hard link not supported ==============
+
+
+            "
+            cat emsdk/upstream/emscripten/emcc > emsdk/upstream/emscripten/em++
+            cat emsdk/upstream/emscripten/emcc > emsdk/upstream/emscripten/cc
+            cat emsdk/upstream/emscripten/emcc.py > emsdk/upstream/emscripten/cc.py
+        fi
 
         cat > emsdk/upstream/emscripten/emar <<END
 #!/bin/bash