Merge branch 'main' into project-filters-hotfix

components/ImagesGallery/ImagesGallery.vue

@@ -47,7 +47,6 @@ import { baseName, extractSection, extractS3BucketName } from '@/utils/common'
  * @param {Number} datasetId
  */
 const getThumbnailData = async (datasetDoi, datasetId, datasetVersion, datasetFacetsData) => {
-  let biolucidaImageData = {}
   let scicrunchData = {}
   let scicrunch_response = []
   let biolucida_response = []
@@ -59,19 +58,15 @@ const getThumbnailData = async (datasetDoi, datasetId, datasetVersion, datasetFa
       biolucida_response = response
     })
 
-    if (biolucida_response.status === 'success') {
-      const scicrunchData = pathOr([], ['data','result'], scicrunch_response)
-      biolucidaImageData = biolucida_response
-      biolucidaImageData['discover_dataset_version'] = datasetVersion
-      biolucidaImageData['dataset_info'] = scicrunchData.length > 0 ? scicrunchData[0] : {}
-    }
-
     if (scicrunch_response.data.result.length > 0) {
       scicrunchData = scicrunch_response.data.result[0]
       scicrunchData.discover_dataset = {
         id: Number(datasetId),
         version: datasetVersion
       }
+      if (biolucida_response.status === 'success') {
+        scicrunchData['dataset_images'] = pathOr([], ['dataset_images'], biolucida_response)
+      }
       // Check for flatmap data
       if (scicrunchData.organs) {
         let flatmapData = []
@@ -133,13 +128,11 @@ const getThumbnailData = async (datasetDoi, datasetId, datasetVersion, datasetFa
       e
     )
     return {
-      biolucidaImageData: {},
       scicrunchData: {},
       hasError: true
     }
   }
   return {
-    biolucidaImageData,
     scicrunchData,
     hasError: false
   }
@@ -213,7 +206,6 @@ export default {
       timeseriesItems: [],
       timeseriesData: [],
       datasetScicrunch: {},
-      datasetBiolucida: {},
     }
   },
   computed: {
@@ -259,13 +251,12 @@ export default {
   },
   async created() {
     this.loading = true
-    const { biolucidaImageData, scicrunchData, hasError } = await getThumbnailData(
+    const { scicrunchData, hasError } = await getThumbnailData(
       this.datasetInfo.doi,
       this.datasetId,
       this.datasetInfo.version,
       this.datasetFacetsData
     )
-    this.datasetBiolucida = biolucidaImageData
     this.datasetScicrunch = scicrunchData
     this.hasError = hasError
 
@@ -324,6 +315,7 @@ export default {
       immediate: true,
       handler: function(scicrunchData) {
         let items = []
+        let bItems = []
         const baseRoute = this.$router.options.base || '/'
         let datasetId = -1
         let datasetVersion = -1
@@ -507,31 +499,29 @@ export default {
           )
         }
         this.scicrunchItems = items
-      }
-    },
-    datasetBiolucida: {
-      deep: true,
-      immediate: true,
-      handler: function(biolucidaData) {
-        const biolucida2DItems = pathOr([], ['dataset_info','biolucida-2d'], biolucidaData)
-        let items = []
-        const baseRoute = this.$router.options.base || '/'
-        if ('dataset_images' in biolucidaData) {
-          items.push(
-            ...Array.from(biolucidaData.dataset_images.filter((obj, index) => {
-              return index === biolucidaData.dataset_images.findIndex(o => obj.image_id === o.image_id);
-            }), dataset_image => {
+
+        if ('dataset_images' in scicrunchData && ('biolucida-2d' in scicrunchData || 'biolucida-3d' in scicrunchData)) {
+          const biolucida2DItems = pathOr([],['biolucida-2d'], scicrunchData)
+          // Images need to exist in both Scicrunch and Biolucida
+          const biolucidaItems = biolucida2DItems.concat(pathOr([],['biolucida-3d'], scicrunchData)).filter((bObject) => {
+            return scicrunchData['dataset_images'].some(image => image.image_id == pathOr("", ['biolucida','identifier'], bObject))
+          })
+          bItems.push(
+            ...Array.from(biolucidaItems, biolucida_item => {
               let filePath = ""
+              const dataset_image = scicrunchData['dataset_images'].find((image) => {
+                return image.image_id == pathOr("", ['biolucida','identifier'], biolucida_item)
+              })
               biolucida2DItems.forEach(biolucida2DItem => {
                 if (pathOr("", ['biolucida','identifier'], biolucida2DItem) == dataset_image.image_id) {
                   filePath = "files/" + pathOr("", ['dataset','path'], biolucida2DItem)
                 }
               })
-              this.getThumbnailFromBiolucida(items, {
+              this.getThumbnailFromBiolucida(bItems, {
                 id: dataset_image.image_id,
                 fetchAttempts: 0
               })
-              this.getImageInfoFromBiolucida(items, {
+              this.getImageInfoFromBiolucida(bItems, {
                 id: dataset_image.image_id,
                 fetchAttempts: 0
               })
@@ -542,16 +532,16 @@ export default {
               // If we can naviagte directly to the file path then do it, otherwise we have to redirect from the datasets/biolucida page
               let linkUrl = filePath != "" ?
                 baseRoute +
-                `datasets/file/${biolucidaData.discover_dataset_id}/${biolucidaData.discover_dataset_version}?path=${filePath}` :
+                `datasets/file/${datasetId}/${datasetVersion}?path=${filePath}` :
                 baseRoute +
                 'datasets/biolucidaviewer/' +
                 dataset_image.image_id +
                 '?view=' +
                 viewEncoding +
                 '&dataset_version=' +
-                biolucidaData.discover_dataset_version +
+                datasetVersion +
                 '&dataset_id=' +
-                biolucidaData.discover_dataset_id +
+                datasetId +
                 '&item_id=' +
                 dataset_image.sourcepkg_id
 
@@ -565,7 +555,7 @@ export default {
             })
           )
         }
-        this.biolucidaItems = items
+        this.biolucidaItems = bItems
       }
     }
   },

nuxt.config.ts

@@ -28,6 +28,7 @@ export default defineNuxtConfig({
         { hid: 'og:image:secure_url', property: 'og:image:secure_url',
           content: 'https://images.ctfassets.net/6bya4tyw8399/7r5WTb92QnHkub8RsExuc1/2ac134de2ddfd65eb6316421df7578f9/sparc-logo-primary.png'
         },
+        { hid: 'robots', name: 'robots', content: 'max-snippet:-1, max-image-preview:large, max-video-preview:-1, crawl-delay:3600' },
         { hid: 'og:url', property: 'og:url', content: process.env.ROOT_URL || 'sparc.science' },
         { hid: 't-type', name: 'twitter:card', content: 'summary_large_image' },
         { name: 'twitter:site', content: '@sparc_science' },
@@ -179,6 +180,7 @@ export default defineNuxtConfig({
       INTERNAL_TRAFFIC_KEY: process.env.INTERNAL_TRAFFIC_KEY || 'internal_traffic',
       INTERNAL_TRAFFIC_VALUE: process.env.INTERNAL_TRAFFIC_VALUE || 'internal',
       SHOW_REHYDRATION_FEATURE: process.env.SHOW_REHYDRATION_FEATURE || 'false',
+      SHOW_DEVICE_TYPE: process.env.SHOW_DEVICE_TYPE || 'false',
       gtm: {
         id: process.env.GOOGLE_TAG_MANAGER_ID || 'GTM-TPT2CVCS',
         defer: true,
@@ -210,20 +212,19 @@ export default defineNuxtConfig({
   },
   robots: {
     sitemap: 'https://sparc.science/sitemap.xml',
-    allow: ['/datasets'],
     // provide simple disallow rules for all robots `user-agent: *`
     // disallowing certain pages that are either redirects, authticated routes, or causing bots to recursively crawl
     disallow: [
       '/welcome', 
       '/user', 
-      '/data', 
       '/contact-us', 
       '/help', 
       '/signup', 
       '/maps',
       '/news-and-events/submit',
       '/news-and-events/community-spotlight/submit'
     ],
-    blockNonSeoBots: true
+    blockNonSeoBots: true,
+    crawlDelay: 3600
   }
 })

pages/data/index.vue

@@ -17,17 +17,19 @@
           Browse categories
         </div>
         <ul class="search-tabs">
-          <li v-for="search in searchTypes" :key="search.label">
-            <nuxt-link class="search-tabs__button" :class="{ active: search.type === $route.query.type }" :to="{
-                name: 'data',
-                query: {
-                  ...$route.query,
-                  type: search.type,
-                }
-              }">
-              {{ search.label }}
-            </nuxt-link>
-          </li>
+          <template v-for="search in searchTypes" :key="search.label">
+            <li v-if="(search.type == 'device' && showDeviceType) || search.type != 'device'">
+              <nuxt-link class="search-tabs__button" :class="{ active: search.type === $route.query.type }" :to="{
+                  name: 'data',
+                  query: {
+                    ...$route.query,
+                    type: search.type,
+                  }
+                }">
+                {{ search.label }}
+              </nuxt-link>
+            </li>
+          </template>
         </ul>
       </div>
       <div class="search-bar__container">
@@ -268,6 +270,9 @@ export default {
   },
 
   computed: {
+    showDeviceType() {
+      return this.$config.public.SHOW_DEVICE_TYPE == 'true'
+    },
     searchType: function () {
       const searchTypeQuery = pathOr('', ['query', 'type'], this.$route)
       const searchType = this.searchTypes.find(searchType => {

server/middleware/webCrawlers.js

@@ -0,0 +1,22 @@
+export default defineEventHandler((event) => {
+  const req = event?.node?.req
+  const res = event?.node?.res
+  const route = event?.node?.url
+  const userAgent = req?.headers['user-agent']?.toLowerCase()
+  const googlebotPaths = ['data', 'datasets']
+  // Remove leading '/' if present
+  let firstPartOfRoute = route?.length > 0 ? route.replace(/^\//, '') : ''
+  const index = firstPartOfRoute.indexOf('/')
+  if (index !== -1) {
+    // Grab the first directory
+    firstPartOfRoute = firstPartOfRoute.substring(0, index);
+  }
+
+  if (firstPartOfRoute == '' || googlebotPaths.includes(firstPartOfRoute)) {
+    if (userAgent && (userAgent.includes('googlebot') || userAgent.includes("google.com/bot.html"))) {
+      res.statusCode = 200
+      res.setHeader('Content-Type', 'text/plain')
+      res.end('Googlebot detected, serving empty response.')
+    }
+  }
+})