Merge pull request #155 from nih-sparc/update_generated_sitemap

Updated sitemap generation and added robots file

nuxt.config.ts

@@ -57,7 +57,8 @@ export default defineNuxtConfig({
     '@pinia-plugin-persistedstate/nuxt',
     '@zadigetvoltaire/nuxt-gtm',
     '@nuxtjs/turnstile',
-    '@nuxtjs/sitemap'
+    '@nuxtjs/sitemap',
+    "nuxt-simple-robots"
   ],
   turnstile: {
     siteKey: process.env.NUXT_PUBLIC_TURNSTILE_SITE_KEY || '0x4AAAAAAATLCwNJ5HNQWRsX'
@@ -200,8 +201,29 @@ export default defineNuxtConfig({
   */
   css: ['sparc-design-system-components-2/dist/style.css', '@/assets/_base.scss'],
   sitemap: {
+    sources: [
+      '/api/__sitemap__/urls'
+    ],
     xslColumns: [
       { label: 'URL', width: '100%' }
     ],
   },
+  robots: {
+    sitemap: 'https://sparc.science/sitemap.xml',
+    allow: ['/datasets'],
+    // provide simple disallow rules for all robots `user-agent: *`
+    // disallowing certain pages that are either redirects, authticated routes, or causing bots to recursively crawl
+    disallow: [
+      '/welcome', 
+      '/user', 
+      '/data', 
+      '/contact-us', 
+      '/help', 
+      '/signup', 
+      '/maps',
+      '/news-and-events/submit',
+      '/news-and-events/community-spotlight/submit'
+    ],
+    blockNonSeoBots: true
+  }
 })

package.json

@@ -18,6 +18,7 @@
     "@zadigetvoltaire/nuxt-gtm": "^0.0.13",
     "cypress": "13.9.0",
     "nuxt": "^3.8.2",
+    "nuxt-simple-robots": "^4.0.0-rc.19",
     "nuxt-svgo": "^3.5.6",
     "sass": "^1.66.1",
     "sitemap": "^7.1.1"

pages/data/index.vue

@@ -6,6 +6,7 @@
     <Meta name="description" hid="description" :content="`Browse ${title}`" />
     <Meta name="og:description" hid="og:description" :content="`Browse ${title}`" />
     <Meta name="twitter:description" :content="`Browse ${title}`" />
+    <Meta name="robots" content="noindex, nofollow" />
   </Head>
   <div class="page-data">
     <breadcrumb :breadcrumb="breadcrumb" :title="searchType.label" />