clean_nhru.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean NHRU Shapefiles to that they can be used to crop an index raster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import geopandas as gpd\n",
    "import subprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "newRegions = ['01','02','03','04','05','06','13','14','15','16','17','18','20','21']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def check_length(fl):\n",
    "    reg = fl.split('/')[-1].split('_')[-1].split('.')[0]\n",
    "    dat = gpd.read_file(fl)\n",
    "    number = len(dat)\n",
    "    print('region %s: %s'%(reg,number))\n",
    "    return reg,number"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "region 01: 2462\n",
      "region 02: 4827\n",
      "region 03: 9899\n",
      "region 04: 5936\n",
      "region 05: 7182\n",
      "region 06: 2303\n",
      "region 07: 8205\n",
      "region 08: 4449\n",
      "region 09: 1717\n",
      "region 10L: 8603\n",
      "region 10U: 10299\n",
      "region 11: 7373\n",
      "region 12: 7815\n",
      "region 13: 1958\n",
      "region 14: 3879\n",
      "region 15: 3441\n",
      "region 16: 2664\n",
      "region 17: 11102\n",
      "region 18: 5837\n",
      "region 20: 322\n",
      "region 21: 489\n"
     ]
    }
   ],
   "source": [
    "regs = []\n",
    "hrus = []\n",
    "for fl in glob.glob('./data/nhrus/nhru_*/nhru_*.shp'):\n",
    "    reg,number = check_length(fl)\n",
    "    regs.append(reg)\n",
    "    hrus.append(number)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame()\n",
    "df['reg'] = regs\n",
    "df['number'] = hrus\n",
    "df.index = df.reg\n",
    "del df['reg']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>number</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reg</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>322</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>09</th>\n",
       "      <td>1717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>06</th>\n",
       "      <td>2303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>01</th>\n",
       "      <td>2462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>3441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>3879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>08</th>\n",
       "      <td>4449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>02</th>\n",
       "      <td>4827</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>5837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>04</th>\n",
       "      <td>5936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>05</th>\n",
       "      <td>7182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>7373</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>7815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>07</th>\n",
       "      <td>8205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10L</th>\n",
       "      <td>8603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>03</th>\n",
       "      <td>9899</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10U</th>\n",
       "      <td>10299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>11102</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     number\n",
       "reg        \n",
       "20      322\n",
       "21      489\n",
       "09     1717\n",
       "13     1958\n",
       "06     2303\n",
       "01     2462\n",
       "16     2664\n",
       "15     3441\n",
       "14     3879\n",
       "08     4449\n",
       "02     4827\n",
       "18     5837\n",
       "04     5936\n",
       "05     7182\n",
       "11     7373\n",
       "12     7815\n",
       "07     8205\n",
       "10L    8603\n",
       "03     9899\n",
       "10U   10299\n",
       "17    11102"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sort_values('number')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def check_val(geom):\n",
    "    return geom.is_valid # return true if geometry is valid in shapely"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def fix_geometry(df):\n",
    "    if df.geometry.is_valid: # if the geometry is valid in shapely, do nothing\n",
    "        return df.geometry\n",
    "    \n",
    "    elif df.geometry.is_valid==False: # if the geometry is invalid, fix it with a buffer of 0\n",
    "        return df.geometry.buffer(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def clean_file(fl):\n",
    "    '''Clean a nhm hru shapefile and save the output'''\n",
    "    print(fl)\n",
    "    dat = gpd.read_file(fl) # load the shapefile\n",
    "    dat['new_geom'] = dat.apply(fix_geometry,axis=1)\n",
    "    dat['valid'] = dat.new_geom.map(check_val)\n",
    "    valid_features = dat.valid.sum()\n",
    "    num_features = len(dat.hru_id_nat.unique())\n",
    "    \n",
    "    if (num_features-valid_features) == 0: # if there are no invalid features\n",
    "        num = fl.split('_')[-2] # get the region code\n",
    "        outfl = './data/nhrus/clean_AEA/nhru_%s_clean.shp'%(num)\n",
    "        dat = dat.set_geometry('new_geom') # assign new geometry to the shapefile\n",
    "        # remove extra columns\n",
    "        del dat['valid']\n",
    "        del dat['geometry']\n",
    "        dat.to_file(outfl) # save the file\n",
    "    else: \n",
    "        return 'error'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def clean_file_12(fl):\n",
    "    '''Clean a nhm hru shapefile and save the output'''\n",
    "    print(fl)\n",
    "    dat = gpd.read_file(fl) # load the shapefile\n",
    "    dat['new_geom'] = dat.apply(fix_geometry,axis=1)\n",
    "    dat['valid'] = dat.new_geom.map(check_val)\n",
    "    valid_features = dat.valid.sum()\n",
    "    num_features = len(dat.hru_id.unique())\n",
    "    \n",
    "    if (num_features-valid_features) == 0: # if there are no invalid features\n",
    "        num = fl.split('_')[-2] # get the region code\n",
    "        outfl = './data/nhrus/clean_AEA/nhru_%s_clean.shp'%(num)\n",
    "        dat = dat.set_geometry('new_geom') # assign new geometry to the shapefile\n",
    "        # remove extra columns\n",
    "        del dat['valid']\n",
    "        del dat['geometry']\n",
    "        dat.to_file(outfl) # save the file\n",
    "    else: \n",
    "        return 'error'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def reproject_shp(fl):\n",
    "    '''reproject the nhm shapefile from WGS84 to EPSG:2163 (Albers Equal Area, National Map)'''\n",
    "    \n",
    "    num = fl.split('_')[-1].split('.')[0]\n",
    "    outfl = './data/nhrus/AEA/nhru_%s_AEA.shp'%(num)\n",
    "    \n",
    "    cmd = \"ogr2ogr -t_srs 'EPSG:2163' -overwrite %s %s\"%(outfl,fl)\n",
    "    subprocess.call(cmd,shell=True)\n",
    "    return None\n",
    "\n",
    "\n",
    "# -s_srs 'EPSG:4326'"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "files = glob.glob('./data/nhrus/nhru_*/nhru_*.shp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# make filenames for the new files:\n",
    "files = []\n",
    "for reg in newRegions:\n",
    "    files.append('./data/nhrus/nhru_%s/nhru_%s.shp'%(reg,reg))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['./data/nhrus/nhru_01/nhru_01.shp',\n",
       " './data/nhrus/nhru_02/nhru_02.shp',\n",
       " './data/nhrus/nhru_03/nhru_03.shp',\n",
       " './data/nhrus/nhru_04/nhru_04.shp',\n",
       " './data/nhrus/nhru_05/nhru_05.shp',\n",
       " './data/nhrus/nhru_06/nhru_06.shp',\n",
       " './data/nhrus/nhru_13/nhru_13.shp',\n",
       " './data/nhrus/nhru_14/nhru_14.shp',\n",
       " './data/nhrus/nhru_15/nhru_15.shp',\n",
       " './data/nhrus/nhru_16/nhru_16.shp',\n",
       " './data/nhrus/nhru_17/nhru_17.shp',\n",
       " './data/nhrus/nhru_18/nhru_18.shp',\n",
       " './data/nhrus/nhru_20/nhru_20.shp',\n",
       " './data/nhrus/nhru_21/nhru_21.shp']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[reproject_shp(fl) for fl in files]"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "files = glob.glob('./data/nhrus/AEA/nhru_*_AEA.shp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# make new files\n",
    "files = []\n",
    "for reg in newRegions:\n",
    "    files.append('./data/nhrus/AEA/nhru_%s_AEA.shp'%reg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['./data/nhrus/AEA/nhru_01_AEA.shp',\n",
       " './data/nhrus/AEA/nhru_02_AEA.shp',\n",
       " './data/nhrus/AEA/nhru_03_AEA.shp',\n",
       " './data/nhrus/AEA/nhru_04_AEA.shp',\n",
       " './data/nhrus/AEA/nhru_05_AEA.shp',\n",
       " './data/nhrus/AEA/nhru_06_AEA.shp',\n",
       " './data/nhrus/AEA/nhru_13_AEA.shp',\n",
       " './data/nhrus/AEA/nhru_14_AEA.shp',\n",
       " './data/nhrus/AEA/nhru_15_AEA.shp',\n",
       " './data/nhrus/AEA/nhru_16_AEA.shp',\n",
       " './data/nhrus/AEA/nhru_17_AEA.shp',\n",
       " './data/nhrus/AEA/nhru_18_AEA.shp',\n",
       " './data/nhrus/AEA/nhru_20_AEA.shp',\n",
       " './data/nhrus/AEA/nhru_21_AEA.shp']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./data/nhrus/AEA/nhru_12_AEA.shp\n"
     ]
    }
   ],
   "source": [
    "clean_file_12(files[-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## This seems to be an issue with region 8 now...."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./data/nhrus/AEA/nhru_01_AEA.shp\n",
      "./data/nhrus/AEA/nhru_02_AEA.shp\n",
      "./data/nhrus/AEA/nhru_03_AEA.shp\n",
      "./data/nhrus/AEA/nhru_04_AEA.shp\n",
      "./data/nhrus/AEA/nhru_05_AEA.shp\n",
      "./data/nhrus/AEA/nhru_06_AEA.shp\n",
      "./data/nhrus/AEA/nhru_13_AEA.shp\n",
      "./data/nhrus/AEA/nhru_14_AEA.shp\n",
      "./data/nhrus/AEA/nhru_15_AEA.shp\n",
      "./data/nhrus/AEA/nhru_16_AEA.shp\n",
      "./data/nhrus/AEA/nhru_17_AEA.shp\n",
      "./data/nhrus/AEA/nhru_18_AEA.shp\n",
      "./data/nhrus/AEA/nhru_20_AEA.shp\n",
      "./data/nhrus/AEA/nhru_21_AEA.shp\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[clean_file(fl) for fl in files]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}