feat: add fetch package

Author: QC-L

package.json

@@ -25,25 +25,25 @@
   },
   "scripts": {
     "clean-dist": "rimraf ./dist",
-    "clean-printable": "rimraf src/content/**/printable.md",
+    "clean-printable": "rimraf src/content/**/printable.mdx",
     "preclean": "run-s clean-dist clean-printable",
-    "clean": "rimraf src/content/**/_*.md src/**/_*.json repositories/*.json",
-    "start": "npm run clean-dist && webpack serve --config webpack.dev.js --env dev --progress --node-env development",
-    "content": "node src/scripts/build-content-tree.js ./src/content ./src/_content.json",
-    "bundle-analyze": "run-s clean fetch content && webpack --config webpack.prod.js --node-env production && run-s printable content && webpack --config webpack.ssg.js --node-env production --env ssg --profile --json > stats.json && webpack-bundle-analyzer stats.json",
-    "fetch-repos": "node src/utilities/fetch-package-repos.js",
+    "clean": "rimraf src/content/**/_*.mdx src/**/_*.json repositories/*.json",
+    "start": "npm run clean-dist && webpack serve --config webpack.dev.mjs --env dev --progress --node-env development",
+    "content": "node src/scripts/build-content-tree.mjs ./src/content ./src/_content.json",
+    "bundle-analyze": "run-s clean fetch content && webpack --config webpack.prod.mjs --node-env production && run-s printable content && webpack --config webpack.ssg.mjs --node-env production --env ssg --profile --json > stats.json && webpack-bundle-analyzer stats.json",
+    "fetch-repos": "node src/utilities/fetch-package-repos.mjs",
     "fetch": "run-p fetch:*",
-    "fetch:readmes": "node src/utilities/fetch-package-readmes.js",
+    "fetch:readmes": "node src/utilities/fetch-package-readmes.mjs",
     "prebuild": "npm run clean",
-    "build": "run-s content && webpack --config webpack.prod.js --node-env production && run-s printable content && webpack --config webpack.ssg.js --node-env production",
+    "build": "run-s content && webpack --config webpack.prod.mjs --node-env production && run-s printable content && webpack --config webpack.ssg.mjs --node-env production --env ssg",
     "postbuild": "npm run sitemap",
     "build-test": "npm run build && http-server --port 4200 dist/",
     "test": "npm run lint",
     "lint": "run-s lint:*",
     "lint:js": "npm run lint-js .",
     "lint-js": "eslint --cache --cache-location .cache/.eslintcache",
-    "lint:markdown": "npm run lint-markdown *.md ./src/content/**/*.md",
-    "lint-markdown": "markdownlint --config ./.markdownlint.json --ignore './src/content/**/_*.md' --ignore '.vale/**/*.md' --ignore '.github/**/*.md'",
+    "lint:markdown": "npm run lint-markdown *.md",
+    "lint-markdown": "markdownlint --config ./.markdownlint.json --ignore '.vale/**/*.md' --ignore '.github/**/*.md'",
     "lint:prose": "vale --config='.vale.ini' src/content",
     "lint:links": "hyperlink -c 8 --root dist -r dist/index.html --canonicalroot https://webpack.js.org/ --internal --skip /plugins/extract-text-webpack-plugin/ --skip /printable --skip https:// --skip http:// --skip sw.js > internal-links.tap; cat internal-links.tap | tap-spot",
     "lint:heading": "textlint --fix src/content/*",
@@ -52,16 +52,16 @@
     "sitemap": "cd dist && sitemap-static --ignore-file=../sitemap-ignore.json --pretty --prefix=https://webpack.js.org/ > sitemap.xml",
     "serve": "npm run build && sirv start ./dist --port 4000",
     "preprintable": "npm run clean-printable",
-    "printable": "node ./src/scripts/concatenate-docs.js",
-    "jest": "jest",
+    "printable": "node ./src/scripts/concatenate-docs.mjs",
+    "jest": "NODE_OPTIONS=--experimental-vm-modules jest --config=jest.config.mjs",
     "cypress:open": "cypress open",
     "cypress:run": "cypress run",
     "cypress:ci": "start-server-and-test build-test http://localhost:4200 cypress:run",
     "prettier": "prettier --write '**/*.{js,json,jsx,css,scss,md,mdx}'",
     "prepare": "husky install"
   },
   "lint-staged": {
-    "*.{js,jsx,md}": [
+    "*.{js,jsx,md,mdx}": [
       "npm run lint-js"
     ],
     "*.md": [
@@ -72,102 +72,99 @@
     ]
   },
   "devDependencies": {
-    "@babel/core": "^7.14.6",
-    "@babel/eslint-parser": "^7.14.5",
+    "@babel/core": "^7.16.0",
+    "@babel/eslint-parser": "^7.16.3",
     "@babel/plugin-proposal-class-properties": "^7.12.1",
-    "@babel/preset-env": "^7.14.5",
-    "@babel/preset-react": "^7.14.5",
-    "@mdx-js/loader": "^1.6.21",
+    "@babel/preset-env": "^7.16.4",
+    "@babel/preset-react": "^7.16.0",
+    "@mdx-js/loader": "^2.0.0-next.9",
     "@octokit/auth-action": "^1.3.3",
-    "@octokit/rest": "^18.6.0",
+    "@octokit/rest": "^18.12.0",
     "@pmmmwh/react-refresh-webpack-plugin": "next",
-    "@svgr/webpack": "^5.5.0",
-    "autoprefixer": "^10.2.6",
-    "babel-loader": "^8.2.2",
-    "copy-webpack-plugin": "^9.0.0",
-    "css-loader": "^5.2.6",
-    "css-minimizer-webpack-plugin": "^3.0.1",
-    "cypress": "^7.5.0",
-    "directory-tree": "^2.2.9",
-    "directory-tree-webpack-plugin": "^1.0.2",
-    "docschina-remark-slugger": "^0.1.4",
+    "@svgr/webpack": "^6.1.1",
+    "autoprefixer": "^10.4.0",
+    "babel-loader": "^8.2.3",
+    "copy-webpack-plugin": "^10.0.0",
+    "css-loader": "^6.5.1",
+    "css-minimizer-webpack-plugin": "^3.2.0",
+    "cypress": "^9.1.1",
+    "directory-tree": "^3.0.1",
+    "directory-tree-webpack-plugin": "^1.0.3",
     "duplexer": "^0.1.1",
-    "eslint": "^7.28.0",
+    "eslint": "^8.4.1",
     "eslint-config-prettier": "^8.3.0",
-    "eslint-plugin-cypress": "^2.11.3",
-    "eslint-plugin-markdown": "^2.2.0",
-    "eslint-plugin-mdx": "^1.13.0",
-    "eslint-plugin-react": "^7.24.0",
-    "eslint-plugin-react-hooks": "^4.2.0",
+    "eslint-plugin-cypress": "^2.12.1",
+    "eslint-plugin-mdx": "^1.16.0",
+    "eslint-plugin-react": "^7.27.1",
+    "eslint-plugin-react-hooks": "^4.3.0",
     "front-matter": "^4.0.2",
+    "github-slugger": "^1.3.0",
     "html-loader": "^2.1.2",
-    "html-webpack-plugin": "^5.3.0",
-    "http-server": "^0.12.3",
-    "husky": "^6.0.0",
-    "hyperlink": "^4.6.1",
-    "jest": "^27.0.4",
-    "lint-staged": "^11.0.0",
+    "html-webpack-plugin": "^5.5.0",
+    "http-server": "^14.0.0",
+    "husky": "^7.0.4",
+    "hyperlink": "^5.0.3",
+    "jest": "^27.4.3",
+    "lint-staged": "^12.1.2",
     "lodash": "^4.17.21",
-    "markdownlint": "^0.23.1",
-    "markdownlint-cli": "^0.27.1",
-    "mini-css-extract-plugin": "^1.6.0",
+    "markdownlint": "^0.24.0",
+    "markdownlint-cli": "^0.30.0",
+    "mdast-util-to-string": "^1.1.0",
+    "mini-css-extract-plugin": "^2.4.5",
     "mkdirp": "^1.0.4",
     "modularscale-sass": "^3.0.3",
-    "node-fetch": "^2.6.1",
+    "node-fetch": "^3.1.0",
     "npm-run-all": "^4.1.1",
-    "postcss": "^8.3.4",
-    "postcss-loader": "^6.1.0",
-    "prettier": "^2.3.1",
-    "react-refresh": "^0.10.0",
+    "postcss": "^8.4.4",
+    "postcss-loader": "^6.2.1",
+    "prettier": "^2.5.1",
+    "react-refresh": "^0.11.0",
     "redirect-webpack-plugin": "^1.0.0",
-    "remark": "^13.0.0",
-    "remark-autolink-headings": "^6.0.1",
-    "remark-emoji": "^2.2.0",
+    "remark": "^14.0.2",
+    "remark-autolink-headings": "7.0.1",
+    "remark-emoji": "^3.0.2",
     "remark-extract-anchors": "1.1.1",
-    "remark-frontmatter": "^3.0.0",
+    "remark-frontmatter": "^4.0.1",
     "remark-gfm": "^1.0.0",
-    "remark-html": "^13.0.1",
-    "remark-loader": "^4.0.0",
-    "remark-refractor": "git://github.com/montogeek/remark-refractor.git",
-    "remark-slug": "^6.0.0",
+    "remark-html": "^15.0.0",
+    "remark-refractor": "montogeek/remark-refractor",
+    "remark-slug": "^7.0.1",
     "rimraf": "^3.0.2",
-    "sass": "^1.35.0",
-    "sass-loader": "^12.1.0",
-    "sirv-cli": "^1.0.12",
+    "sass": "^1.44.0",
+    "sass-loader": "^12.4.0",
+    "sirv-cli": "^1.0.14",
     "sitemap-static": "^0.4.2",
-    "start-server-and-test": "^1.12.5",
+    "start-server-and-test": "^1.14.0",
     "static-site-generator-webpack-plugin": "^3.4.1",
-    "style-loader": "^2.0.0",
-    "tailwindcss": "^2.1.4",
+    "style-loader": "^3.3.1",
+    "tailwindcss": "^3.0.0",
     "tap-spot": "^1.1.1",
     "textlint": "^11.8.2",
     "textlint-rule-heading": "^1.0.10",
-    "unist-util-visit": "^2.0.3",
-    "webpack": "^5.39.0",
-    "webpack-bundle-analyzer": "^4.4.2",
-    "webpack-cli": "^4.7.2",
-    "webpack-dev-server": "^4.0.0-beta.3",
+    "unist-util-visit": "^4.1.0",
+    "webpack": "^5.65.0",
+    "webpack-bundle-analyzer": "^4.5.0",
+    "webpack-cli": "^4.9.1",
+    "webpack-dev-server": "^4.6.0",
     "webpack-merge": "^5.8.0",
-    "workbox-webpack-plugin": "^6.1.5"
+    "workbox-webpack-plugin": "^6.4.2"
   },
   "dependencies": {
-    "docsearch.js": "^2.5.2",
+    "@docsearch/react": "^3.0.0-alpha.42",
     "path-browserify": "^1.0.1",
     "prop-types": "^15.7.2",
     "react": "^17.0.2",
-    "react-banner": "^1.0.0-rc.0",
     "react-dom": "^17.0.2",
     "react-g-analytics": "0.4.2",
-    "react-helmet-async": "^1.0.9",
-    "react-router-dom": "^5.2.0",
-    "react-spring": "next",
+    "react-helmet-async": "^1.2.2",
+    "react-router-dom": "^5.3.0",
+    "react-spring": "^9.3.2",
     "react-tiny-popover": "5",
-    "react-use": "^17.2.4",
+    "react-use": "^17.3.1",
     "react-visibility-sensor": "^5.0.2",
-    "unist-util-visit": "^2.0.3",
     "webpack-pwa-manifest": "^4.3.0",
     "webpack.vote": "https://github.com/webpack/voting-app.git",
-    "workbox-window": "^6.1.5"
+    "workbox-window": "^6.4.2"
   },
   "resolutions": {
     "sitemap-static/minimist": "1.2.5",

src/utilities/constants.js renamed to src/utilities/constants.mjs

@@ -1,4 +1,4 @@
-const excludedLoaders = [
+export const excludedLoaders = [
   'webpack-contrib/config-loader',
   'webpack-contrib/eslint-loader',
   'webpack-contrib/transform-loader',
@@ -25,7 +25,7 @@ const excludedLoaders = [
   'webpack-contrib/istanbul-instrumenter-loader',
   'webpack-contrib/worker-loader',
 ];
-const excludedPlugins = [
+export const excludedPlugins = [
   'webpack-contrib/component-webpack-plugin',
   'webpack-contrib/extract-text-webpack-plugin',
   'webpack-contrib/i18n-webpack-plugin',
@@ -34,8 +34,3 @@ const excludedPlugins = [
   'webpack-contrib/zopfli-webpack-plugin',
   'webpack-contrib/closure-webpack-plugin',
 ];
-
-module.exports = {
-  excludedLoaders,
-  excludedPlugins,
-};

src/utilities/fetch-package-readmes.mjs

@@ -0,0 +1,97 @@
+import _ from 'lodash';
+import fs from 'fs';
+import path from 'path';
+import { promisify } from 'util';
+import mkdirp from 'mkdirp';
+import fetch from 'node-fetch';
+import { fileURLToPath } from 'url';
+
+import yamlHeadmatter from './yaml-headmatter.mjs';
+import processReadme from './process-readme.mjs';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const writeFile = promisify(fs.writeFile);
+const rename = promisify(fs.rename);
+const readFile = promisify(fs.readFile);
+const cwd = process.cwd();
+
+const types = ['loaders', 'plugins'];
+
+const pathMap = {
+  loaders: path.resolve(__dirname, '../content/loaders'),
+  plugins: path.resolve(__dirname, '../content/plugins'),
+};
+
+const loaderGroup = {
+  'css-loader': 'CSS',
+  'less-loader': 'CSS',
+  'postcss-loader': 'CSS',
+  'sass-loader': 'CSS',
+  'style-loader': 'CSS',
+  'stylus-loader': 'CSS',
+};
+
+async function main() {
+  for (const type of types) {
+    const outputDir = pathMap[type];
+
+    await mkdirp(outputDir);
+
+    const repos = JSON.parse(
+      await readFile(path.resolve(__dirname, `../../repositories/${type}.json`))
+    );
+
+    for (const repo of repos) {
+      const [, packageName] = repo.split('/');
+      const url = `https://raw.githubusercontent.com/${repo}/master/README.md`;
+      const htmlUrl = `https://github.com/${repo}`;
+      const editUrl = `${htmlUrl}/edit/master/README.md`;
+      const fileName = path.resolve(outputDir, `${packageName}.mdx`);
+      const mdxFileName = path.resolve(outputDir, `${packageName}.mdx`);
+
+      let title = packageName;
+
+      if (type === 'plugins') {
+        title = _.camelCase(title);
+        title = _.upperFirst(title);
+        title = title.replace(/I18N/, 'I18n');
+      }
+
+      // generate yaml matter for file
+      let headmatter;
+
+      if (type === 'plugins') {
+        headmatter = yamlHeadmatter({
+          title: title,
+          group: 'webpack contrib',
+          source: url,
+          edit: editUrl,
+          repo: htmlUrl,
+        });
+      } else {
+        let basic = {
+          title: title,
+          source: url,
+          edit: editUrl,
+          repo: htmlUrl,
+        };
+
+        if (loaderGroup[packageName]) {
+          basic.group = loaderGroup[packageName];
+        }
+        headmatter = yamlHeadmatter(basic);
+      }
+
+      const response = await fetch(url);
+      const content = await response.text();
+      const body = processReadme(content, { source: url });
+      await writeFile(fileName, headmatter + body);
+      await rename(fileName, mdxFileName);
+      console.log('Generated:', path.relative(cwd, mdxFileName));
+    }
+  }
+}
+
+main();