updated site-mapping

Author: justinTsugranes

README.md

@@ -24,6 +24,7 @@ Here are some of the core dependencies and features:
   - [Remark](https://www.npmjs.com/package/remark-package-dependencies) - Inject to markdown the list of dependencies of your package
   - [Remark MDX](https://www.npmjs.com/package/remark-mdx) - Remark plugin to support the MDX syntax
   - [date-fns](https://date-fns.org/v2.16.1/docs/format) - Return the formatted date string in the given format
+- [site-map](https://github.com/iamvishnusankar/next-sitemap) - Sitemap generator for next.js. Generate sitemap(s) and robots.txt for all static/pre-rendered/dynamic/server-side pages.
 - [robots.txt](https://developers.google.com/search/docs/advanced/robots/intro) - Tells search engine crawlers which URLs the crawler can access on your site.
 
 ### To Use

package-lock.json

@@ -6,6 +6,7 @@
   "scripts": {
     "dev": "next dev",
     "build": "next build",
+    "postbuild": "next-sitemap",
     "format": "prettier --ignore-path .gitignore --write .",
     "start": "next start",
     "export": "next build && next export",
@@ -60,6 +61,7 @@
     "eslint-plugin-react-hooks": "^4.3.0",
     "jest": "^28.1.3",
     "jest-environment-jsdom": "^28.1.3",
+    "next-sitemap": "^3.1.17",
     "postcss": "^8.4.16",
     "postcss-preset-env": "^7.7.2",
     "prettier": "^2.7.1",

package.json

@@ -0,0 +1,8 @@
+/** @type {import('next-sitemap').IConfig} */
+const config = {
+  siteUrl: process.env.SITE_URL || 'https://example.com',
+  generateRobotsTxt: true, // (optional)
+  // ...other options
+}
+
+export default config

public/next-sitemap.config.js

@@ -1,14 +1,14 @@
+# Block all crawlers for /accounts
+User-agent: *
+Disallow: /accounts
+
+#  Googlebot is not allowed to crawl any URL that starts with http://example.com/nogooglebot/
 User-agent: Googlebot
 Disallow: /nogooglebot/
 
+# Allow all crawlers - All other user agents are allowed to crawl the entire site.
 User-agent: *
 Allow: /
 
-Sitemap: http://www.example.com/sitemap.xml
-
-
-/*
-1. The user agent named Googlebot is not allowed to crawl any URL that starts with http://example.com/nogooglebot/.
-2. All other user agents are allowed to crawl the entire site. This could have been omitted and the result would be the same; the default behavior is that user agents are allowed to crawl the entire site.
-3. The site's sitemap file is located at http://www.example.com/sitemap.xml.
-*/
+# The site's sitemap file is located at:
+Sitemap: http://www.SITE.com/sitemap.xml

public/robots.txt

@@ -322,6 +322,11 @@
   "resolved" "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz"
   "version" "1.5.0"
 
+"@corex/deepmerge@^4.0.29":
+  "integrity" "sha512-q/yVUnqckA8Do+EvAfpy7RLdumnBy9ZsducMUtZTvpdbJC7azEf1hGtnYYxm0QfphYxjwggv6XtH64prvS1W+A=="
+  "resolved" "https://registry.npmjs.org/@corex/deepmerge/-/deepmerge-4.0.29.tgz"
+  "version" "4.0.29"
+
 "@cspotcode/source-map-support@^0.8.0":
   "integrity" "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw=="
   "resolved" "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz"
@@ -834,7 +839,7 @@
     "@types/mdx" "^2.0.0"
     "@types/react" ">=16"
 
-"@next/env@12.2.4":
+"@next/env@*", "@next/env@12.2.4":
   "integrity" "sha512-/gApFXWk5CCLFQJL5IYJXxPQuG5tz5nPX4l27A9Zm/+wJxiwFrRSP54AopDxIv4JRp/rGwcgk/lZS/0Clw8jYA=="
   "resolved" "https://registry.npmjs.org/@next/env/-/env-12.2.4.tgz"
   "version" "12.2.4"
@@ -5300,7 +5305,15 @@
   "resolved" "https://registry.npmjs.org/next-seo/-/next-seo-5.5.0.tgz"
   "version" "5.5.0"
 
-"next@^8.1.1-canary.54 || >=9.0.0", "next@latest":
+"next-sitemap@^3.1.17":
+  "integrity" "sha512-xop7KgEbWsqOe4Fr50g9RQ1UGb1bAEJoKYVVynbUqietpSltAnCIzvq/StL2gZcNyFW9K+9tZV7EpcD+72gRrA=="
+  "resolved" "https://registry.npmjs.org/next-sitemap/-/next-sitemap-3.1.17.tgz"
+  "version" "3.1.17"
+  dependencies:
+    "@corex/deepmerge" "^4.0.29"
+    "minimist" "^1.2.6"
+
+"next@*", "next@^8.1.1-canary.54 || >=9.0.0", "next@latest":
   "integrity" "sha512-b1xlxEozmAWokAXzXsi5vlmU/IfJcFNIJA8dpU5UdkFbyDPio8wwb8mAQ/Y7rGtfTgG/t/u49BiyEA+xAgFvow=="
   "resolved" "https://registry.npmjs.org/next/-/next-12.2.4.tgz"
   "version" "12.2.4"