Skip to content

Commit

Permalink
DRILL-7063: Update addremove_files.sh (#575)
Browse files Browse the repository at this point in the history
  • Loading branch information
@agirish
agirish authored Apr 10, 2019
1 parent c018fd7 commit 7d81493
Showing 1 changed file with 16 additions and 11 deletions.
27 changes: 16 additions & 11 deletions framework/resources/Datasources/metadata_caching/addremove_files.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,24 +26,20 @@ fi
if [ $removeDirExists -eq 0 ]; then
hadoop fs -rm -r /drill/testdata/metadata_caching/lineitem_removedir
fi
if [ addAutoPartitionExists -eq 0 ]; then
if [ $addAutoPartitionExists -eq 0 ]; then
hadoop fs -rm -r /drill/testdata/metadata_caching/lineitem_addautopartitioned_files
fi
if [ $removeAutoPartitionExists -eq 0 ]; then
hadoop fs -rm -r /drill/testdata/metadata_caching/lineitem_removeautopartitioned_files
fi




hadoop fs -copyFromLocal ${DRILL_TEST_DATA_DIR}/Datasources/metadata_caching/data/lineitem_addfiles /drill/testdata/metadata_caching/
hadoop fs -copyFromLocal ${DRILL_TEST_DATA_DIR}/Datasources/metadata_caching/data/lineitem_removefiles /drill/testdata/metadata_caching/
hadoop fs -copyFromLocal ${DRILL_TEST_DATA_DIR}/Datasources/metadata_caching/data/lineitem_adddir /drill/testdata/metadata_caching/
hadoop fs -copyFromLocal ${DRILL_TEST_DATA_DIR}/Datasources/metadata_caching/data/lineitem_removedir /drill/testdata/metadata_caching/
hadoop fs -copyFromLocal ${DRILL_TEST_DATA_DIR}/Datasources/metadata_caching/data/lineitem_addautopartitioned_files /drill/testdata/metadata_caching/
hadoop fs -copyFromLocal ${DRILL_TEST_DATA_DIR}/Datasources/metadata_caching/data/lineitem_removeautopartitioned_files /drill/testdata/metadata_caching/


${DRILL_TEST_DATA_DIR}/Datasources/metadata_caching/delete_cache.sh /drill/testdata/metadata_caching/lineitem_addfiles
${DRILL_TEST_DATA_DIR}/Datasources/metadata_caching/delete_cache.sh /drill/testdata/metadata_caching/lineitem_removefiles
${DRILL_TEST_DATA_DIR}/Datasources/metadata_caching/delete_cache.sh /drill/testdata/metadata_caching/lineitem_adddir
Expand All @@ -59,11 +55,11 @@ else
${DRILL_HOME}/bin/sqlline -n ${USERNAME} -p ${PASSWORD} -u "jdbc:drill:schema=dfs.$1;drillbit=${DRILL_STORAGE_PLUGIN_SERVER}" --run=${DRILL_TEST_DATA_DIR}/Datasources/metadata_caching/refresh_metadata_addremovefiles.ddl
fi

hadoop fs -copyFromLocal ${DRILL_TEST_DATA_DIR}/Datasources/metadata_caching/data/lineitem1.parquet /drill/testdata/metadata_caching/lineitem_addfiles/lineitem1.parquet
hadoop fs -copyFromLocal -f ${DRILL_TEST_DATA_DIR}/Datasources/metadata_caching/data/lineitem1.parquet /drill/testdata/metadata_caching/lineitem_addfiles/lineitem1.parquet
hadoop fs -rm /drill/testdata/metadata_caching/lineitem_removefiles/lineitem1.parquet
hadoop fs -copyFromLocal ${DRILL_TEST_DATA_DIR}/Datasources/metadata_caching/data/feb /drill/testdata/metadata_caching/lineitem_adddir/
hadoop fs -copyFromLocal -f ${DRILL_TEST_DATA_DIR}/Datasources/metadata_caching/data/feb /drill/testdata/metadata_caching/lineitem_adddir/
hadoop fs -rm -r /drill/testdata/metadata_caching/lineitem_removedir/feb
hadoop fs -copyFromLocal ${DRILL_TEST_DATA_DIR}/Datasources/metadata_caching/data/lineitem2.parquet /drill/testdata/metadata_caching/lineitem_addautopartitioned_files/
hadoop fs -copyFromLocal -f ${DRILL_TEST_DATA_DIR}/Datasources/metadata_caching/data/lineitem2.parquet /drill/testdata/metadata_caching/lineitem_addautopartitioned_files/
hadoop fs -rm /drill/testdata/metadata_caching/lineitem_removeautopartitioned_files/lineitem2.parquet

# copy the generated cache file and then validate its contents in the testcases
Expand All @@ -74,6 +70,15 @@ if [ $generatedCachesExists -eq 0 ]; then
hadoop fs -rm -r /drill/testdata/metadata_caching/generated_caches
fi
hadoop fs -mkdir -p /drill/testdata/metadata_caching/generated_caches
hadoop fs -cp /drill/testdata/metadata_caching/fewtypes/.drill.parquet_metadata /drill/testdata/metadata_caching/generated_caches/fewtypes_cache.json
hadoop fs -cp /drill/testdata/metadata_caching/fewtypes_varcharpartition/.drill.parquet_metadata /drill/testdata/metadata_caching/generated_caches/fewtypes_varcharpartition_cache.json
hadoop fs -cp /drill/testdata/metadata_caching/lineitem/.drill.parquet_metadata /drill/testdata/metadata_caching/generated_caches/lineitem_cache.json
hadoop fs -mkdir -p /drill/testdata/metadata_caching/generated_caches/fewtypes_cache
hadoop fs -cp -f /drill/testdata/metadata_caching/fewtypes/.drill.parquet_file_metadata.v4 /drill/testdata/metadata_caching/generated_caches/fewtypes_cache/parquet_file_metadata.json
hadoop fs -cp -f /drill/testdata/metadata_caching/fewtypes/.drill.parquet_summary_metadata.v4 /drill/testdata/metadata_caching/generated_caches/fewtypes_cache/parquet_summary_metadata.json

hadoop fs -mkdir -p /drill/testdata/metadata_caching/generated_caches/fewtypes_varcharpartition_cache
hadoop fs -cp -f /drill/testdata/metadata_caching/fewtypes_varcharpartition/.drill.parquet_file_metadata.v4 /drill/testdata/metadata_caching/generated_caches/fewtypes_varcharpartition_cache/parquet_file_metadata.json
hadoop fs -cp -f /drill/testdata/metadata_caching/fewtypes_varcharpartition/.drill.parquet_summary_metadata.v4 /drill/testdata/metadata_caching/generated_caches/fewtypes_varcharpartition_cache/parquet_summary_metadata.json

hadoop fs -mkdir -p /drill/testdata/metadata_caching/generated_caches/lineitem_cache
hadoop fs -cp -f /drill/testdata/metadata_caching/lineitem/.drill.parquet_file_metadata.v4 /drill/testdata/metadata_caching/generated_caches/lineitem_cache/parquet_file_metadata.json
hadoop fs -cp -f /drill/testdata/metadata_caching/lineitem/.drill.parquet_summary_metadata.v4 /drill/testdata/metadata_caching/generated_caches/lineitem_cache/parquet_summary_metadata.json
hadoop fs -cp -f /drill/testdata/metadata_caching/lineitem/.drill.parquet_metadata_directories /drill/testdata/metadata_caching/generated_caches/lineitem_cache/parquet_metadata_directories

0 comments on commit 7d81493

Please sign in to comment.