add-CVE-URL-CSV.sh

#!/bin/bash

# Run from /gsd-database/ root directory

# Takes one argument of a CSV file with CVE,URL

input=$1

while IFS= read -r line
do
  # Remove everything after the first comma to get CVE
  CVE_ID=`echo $line | sed 's/,.*//'`
  # TODO: add checks to ensure it is actually a valid CVE and error out if not
#  if [[ $DATA =~ CVE-[1-2][0-9][0-9][0-9]-[0-9]+ ]]
#    then
#      continue
#  fi

  # Remove everything after the last comma (should only be one) to get the URL
  URL=`echo $line | sed 's/^.*,//'`


  GSD_ID=`echo $CVE_ID | sed 's/CVE/GSD/'`
  YEAR=`echo $CVE_ID | cut -d"-" -f2`
  INT=`echo $CVE_ID | cut -d"-" -f3`
  DIR=`echo $INT | sed 's/[0-9][0-9][0-9]$/xxx/'`
  FILENAME="$GSD_ID.json"
  FILEDIR="$YEAR/$DIR/"
  FILE="$YEAR/$DIR/$FILENAME"

  #
  # Check year for correctness
  #


  #
  # Check INT for correctness, 4+ digits.
  #
  if [[ $INT =~ ^[0-9][0-9][0-9][0-9]+$ ]]
  then
    #  3. If no file is found create a stub file and add the URL. Please note that some vendors have errors/typos in their data, as we discover these we will clean up the files we have created. I have already contacted vendors about cleaning these up.

      # Check if we have the file for that ID
      if [ -f "$FILE" ]; then
        #  1. If one is found with the URL, great, ignore, we already have it.
        # Check if the key already exists
        FOUND="true"
        # If no GSD.references[] array exiosts you'll get an error
        FOUND=`jq -e '.GSD.references|any(. == "'$URL'")' $FILE`
        if [ $FOUND == "true" ]; then
          # Move along nothing to see here, unless you want to add dots as a done-o-dial, it can be soothing to watch
          echo -n ""

        else
          #  2. If one is found with no URL, add the URL.
          TEMP=$(mktemp)
          # So if false or null we add the url
          # jq converts flopats to ints, e.g. 10.0 to 10
          jq --arg new "$URL" '.GSD.references? += [$new]' $FILE > $TEMP
          # print fix, assumes repo location
          ../gsd-tools/local-scripts/print-json.py $TEMP
          # fix float to int problem intentioanlly obtuse to be readable
          # ": 6.0,
          # ": 10.0,
          sed 's/": 1,$/": 1.0,/' $TEMP | sed 's/": 2,$/": 2.0,/' | sed 's/": 3,$/": 3.0,/' | sed 's/": 4,$/": 4.0,/' | sed 's/": 5,$/": 5.0,/' | sed 's/": 6,$/": 6.0,/' | sed 's/": 7,$/": 7.0,/' | sed 's/": 8,$/": 8.0,/' | sed 's/": 9,$/": 9.0,/' | sed 's/": 10,$/": 10.0,/' | sed 's/": 1$/": 1.0/' | sed 's/": 2$/": 2.0/' | sed 's/": 3$/": 3.0/' | sed 's/": 4$/": 4.0/' | sed 's/": 5$/": 5.0/' | sed 's/": 6$/": 6.0/' | sed 's/": 7$/": 7.0/' | sed 's/": 8$/": 8.0/' | sed 's/": 9$/": 9.0/' | sed 's/": 0$/": 10.0/' > $FILE
          rm -f $TEMP
          # done-o-dial
          echo "UPDATED: $GSD_ID"
        fi

        else
          #  3. If no file is found create a stub file and add the URL. Please note that some vendors have errors/typos in their data, as we discover these we will clean up the files we have created. I have already contacted vendors about cleaning these up.
          #{
          #    "GSD": {
          #        "alias": "CVE-YEAR-INTEGER",
          #        "id": "GSD-YEAR-INTEGER",
          #        "references": [
          #            "https://VENDOR_LINK/HERE"
          #        ]
          #    }
          #}
          # They may not have the directory
          mkdir -p $FILEDIR
          touch $FILE

          TEMP=$(mktemp)
          # Create stub file
          echo -n '{  "GSD": {    "alias": "' > $TEMP
          echo -n "$CVE_ID" >> $TEMP
          echo -n '",  "id": "' >> $TEMP
          echo -n "$GSD_ID" >> $TEMP
          echo -n '",    "references": [      "' >> $TEMP
          echo -n "$URL" >> $TEMP
          echo -n '"    ]  }}' >> $TEMP

          # pretty print it
          ../gsd-tools/local-scripts/print-json.py $TEMP
          # fix float to int problem intentioanlly obtuse to be readable
          # ": 6.0,
          # ": 10.0,
          sed 's/": 1,$/": 1.0,/' $TEMP | sed 's/": 2,$/": 2.0,/' | sed 's/": 3,$/": 3.0,/' | sed 's/": 4,$/": 4.0,/' | sed 's/": 5,$/": 5.0,/' | sed 's/": 6,$/": 6.0,/' | sed 's/": 7,$/": 7.0,/' | sed 's/": 8,$/": 8.0,/' | sed 's/": 9,$/": 9.0,/' | sed 's/": 10,$/": 10.0,/' | sed 's/": 1$/": 1.0/' | sed 's/": 2$/": 2.0/' | sed 's/": 3$/": 3.0/' | sed 's/": 4$/": 4.0/' | sed 's/": 5$/": 5.0/' | sed 's/": 6$/": 6.0/' | sed 's/": 7$/": 7.0/' | sed 's/": 8$/": 8.0/' | sed 's/": 9$/": 9.0/' | sed 's/": 0$/": 10.0/' > $FILE
          rm -f $TEMP
          # done-o-dial
          echo "CREATED FILE: $GSD_ID"
      fi
  else
      echo "BAD INTEGER IN CVE ID"
  fi



done < $input

exit