♻️ Make "go to source definition" work (#658)

Sorry for all the codeowners reviews

Make "go to source definition" work by using the package, thanks to
`.d.ts.map` files generated by `tsc`.

Defer type generation to `tsc` instead of `tsup`, since `tsup` is not
able to generate `.d.ts.map`. Also removed `shared` package - since we
don't want to publish it, having "go to source" with it would have been
problematic. Instead, "shared" files are duplicated between
`@huggingface/inference` and `@huggingface/hub`.

Maybe later we can create `@huggingface/webblob` and
`@huggingface/fileblob` if we want to reuse them.

See also: https://www.npmjs.com/package/dts-buddy used by svelte-kit

Author: coyotte508

.github/workflows/hub-publish.yml

@@ -52,6 +52,44 @@ jobs:
           git add ../..
           git commit -m "🔖 @hugginface/hub $BUMPED_VERSION"
           git tag "hub-v$BUMPED_VERSION"
+
+      - name: Make sure that the latest version of @huggingface/tasks is consistent with the local version
+        run: |
+          LOCAL_TASKS_VERSION=$(node -p "require('./package.json').version")
+          REMOTE_TASKS_VERSION=$(npm view @huggingface/tasks version)
+
+          # If the versions are different, error 
+          if [ "$LOCAL_TASKS_VERSION" != "$REMOTE_TASKS_VERSION" ]; then
+            echo "Error: The local @huggingface/tasks package version ($LOCAL_TASKS_VERSION) differs from the remote version ($REMOTE_TASKS_VERSION). Release halted."
+            exit 1
+          fi
+
+          npm pack @huggingface/tasks
+          mv huggingface-tasks-$LOCAL_TASKS_VERSION.tgz tasks-local.tgz
+
+          npm pack @huggingface/tasks@$REMOTE_TASKS_VERSION
+          mv huggingface-tasks-$REMOTE_TASKS_VERSION.tgz tasks-remote.tgz
+
+          # Compute checksum of local tar. We need to extract both tar since the remote compression might be different
+          tar -xf tasks-local.tgz
+          LOCAL_CHECKSUM=$(cd package && tar --mtime='1970-01-01' --mode=755 -cf - . | sha256sum | cut -d' ' -f1)
+          echo "Local package checksum: $LOCAL_CHECKSUM"
+
+          rm -Rf package
+
+          tar -xf tasks-remote.tgz
+          REMOTE_CHECKSUM=$(cd package && tar --mtime='1970-01-01' --mode=755 -cf - . | sha256sum | cut -d' ' -f1)
+          echo "Remote package checksum: $REMOTE_CHECKSUM"
+
+          rm -Rf package
+
+          if [ "$LOCAL_CHECKSUM" != "$REMOTE_CHECKSUM" ]; then
+            echo "Checksum Verification Failed: The local @huggingface/tasks package differs from the remote version. Release halted. Local Checksum: $LOCAL_CHECKSUM, Remote Checksum: $REMOTE_CHECKSUM"
+            exit 1
+          fi
+          echo "Checksum Verification Successful: The local and remote @huggingface/tasks packages are consistent. Proceeding with the @huggingface/widgets package release. Local Checksum: $LOCAL_CHECKSUM, Remote Checksum: $REMOTE_CHECKSUM."
+        working-directory: packages/tasks
+
       - run: pnpm publish --no-git-checks .
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/test.yml

@@ -6,7 +6,7 @@ on:
       - main
 
 jobs:
-  test:
+  node:
     runs-on: ubuntu-latest
     timeout-minutes: 10
 
@@ -42,11 +42,57 @@ jobs:
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
 
+  browser:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          # Could use something like rmacklin/fetch-through-merge-base@v0 instead when the repo gets bigger
+          fetch-depth: 0
+      - name: "Extracting the merge base into 'SINCE'"
+        id: since
+        run: |
+          if [ -z "${{  github.event.pull_request.head.ref }}" ]
+          then
+            echo "SINCE=${{ github.sha }}^1" >> $GITHUB_OUTPUT
+          else
+            echo "SINCE=$(git merge-base origin/${{ github.event.pull_request.base.ref }} ${{ github.sha }})" >> $GITHUB_OUTPUT
+          fi
+
+      - run: corepack enable
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+          cache: "pnpm"
+          cache-dependency-path: "**/pnpm-lock.yaml"
+      - run: |
+          pnpm install --frozen-lockfile --filter .
+          pnpm install --frozen-lockfile --filter ...[${{ steps.since.outputs.SINCE }}]...
+          pnpm --filter ...[${{ steps.since.outputs.SINCE }}]... build
+
       - name: Test in browser
         run: VCR_MODE=playback pnpm --filter ...[${{ steps.since.outputs.SINCE }}] test:browser
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
 
+  e2e:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - run: corepack enable
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+          cache: "pnpm"
+          cache-dependency-path: "**/pnpm-lock.yaml"
+
       - name: E2E - start mock npm registry
         run: |
           npm i -g verdaccio verdaccio-memory verdaccio-auth-memory
@@ -58,7 +104,7 @@ jobs:
         working-directory: e2e
         run: |
           sleep 3
-          pnpm i --filter inference... --filter hub... --frozen-lockfile
+          pnpm i --filter root --filter inference... --filter hub... --frozen-lockfile
           pnpm --filter inference --filter hub --filter tasks publish --force --no-git-checks --registry http://localhost:4874/
 
       - name: E2E test - test yarn install

e2e/.gitignore

@@ -0,0 +1,2 @@
+# Modified during E2E tests
+.npmrc

package.json

@@ -1,4 +1,6 @@
 {
+	"private": true,
+	"name": "@huggingface/root",
 	"license": "MIT",
 	"packageManager": "pnpm@8.10.5",
 	"scripts": {

packages/agents/package.json

@@ -31,7 +31,7 @@
 		"format": "prettier --write .",
 		"format:check": "prettier --check .",
 		"prepublishOnly": "pnpm run build",
-		"build": "tsup",
+		"build": "tsup && tsc --emitDeclarationOnly --declaration",
 		"test": "vitest run",
 		"test:browser": "vitest run --browser.name=chrome --browser.headless --config vitest-browser.config.mts",
 		"check": "tsc"
@@ -53,8 +53,7 @@
 	"author": "Hugging Face",
 	"license": "MIT",
 	"devDependencies": {
-		"@types/node": "^18.13.0",
-		"type-fest": "^3.9.0"
+		"@types/node": "^18.13.0"
 	},
 	"dependencies": {
 		"@huggingface/inference": "^2.6.1"

packages/agents/pnpm-lock.yaml

@@ -12,8 +12,9 @@
 		"skipLibCheck": true,
 		"noImplicitOverride": true,
 		"outDir": "./dist",
-		"declaration": true
+		"declaration": true,
+		"declarationMap": true
 	},
-	"include": ["src", "test", "index.ts", "../shared/src"],
+	"include": ["src", "test", "index.ts"],
 	"exclude": ["dist"]
 }

packages/agents/tsconfig.json

@@ -5,9 +5,6 @@ const baseConfig: Options = {
 	format: ["cjs", "esm"],
 	outDir: "dist",
 	clean: true,
-	dts: {
-		resolve: true,
-	},
 };
 
 const nodeConfig: Options = {

packages/agents/tsup.config.ts

@@ -0,0 +1 @@
+dist

packages/gguf/.eslintignore

@@ -31,7 +31,7 @@
 		"format": "prettier --write .",
 		"format:check": "prettier --check .",
 		"prepublishOnly": "pnpm run build",
-		"build": "tsup src/index.ts --format cjs,esm --clean --dts",
+		"build": "tsup src/index.ts --format cjs,esm --clean && tsc --emitDeclarationOnly --declaration",
 		"build:llm": "tsx scripts/generate-llm.ts && pnpm run format",
 		"test": "vitest run",
 		"check": "tsc"
@@ -47,8 +47,5 @@
 		"gguf"
 	],
 	"author": "Hugging Face",
-	"license": "MIT",
-	"devDependencies": {
-		"type-fest": "^3.9.0"
-	}
+	"license": "MIT"
 }