diff --git a/conf/plugin-templates/common/dfs-storage-plugin.template b/conf/plugin-templates/common/dfs-storage-plugin.template index 42149c728..600a1c05b 100755 --- a/conf/plugin-templates/common/dfs-storage-plugin.template +++ b/conf/plugin-templates/common/dfs-storage-plugin.template @@ -547,6 +547,12 @@ "extensions" : [ "csv" ], "delimiter" : "," }, + "csvh": { + "type": "text", + "extensions": [ "csvh" ], + "extractHeader": true, + "delimiter": "," + }, "tsv" : { "type" : "text", "extensions" : [ "tsv" ], diff --git a/framework/resources/Datasources/text_storage/v3_text_reader/region_malformed.csvh b/framework/resources/Datasources/text_storage/v3_text_reader/region_malformed.csvh new file mode 100644 index 000000000..0097e53fa --- /dev/null +++ b/framework/resources/Datasources/text_storage/v3_text_reader/region_malformed.csvh @@ -0,0 +1,6 @@ +id,region,comment +0,AFRICA,lar deposits. blithely final packages cajole. regular waters are final requests. regular accounts are according to + +2,ASIA,ges. thinly even pinto beans ca +3,EUROPE, +4,MIDDLE EAST \ No newline at end of file diff --git a/framework/resources/Datasources/text_storage/v3_text_reader/region_partitioned/nested/region_2.tbl b/framework/resources/Datasources/text_storage/v3_text_reader/region_partitioned/nested/region_2.tbl new file mode 100644 index 000000000..17441bf96 --- /dev/null +++ b/framework/resources/Datasources/text_storage/v3_text_reader/region_partitioned/nested/region_2.tbl @@ -0,0 +1,5 @@ +0|AFRICA|lar deposits. blithely final packages cajole. regular waters are final requests. regular accounts are according to +1|AMERICA|hs use ironic, even requests. s +2|ASIA|ges. thinly even pinto beans ca +3|EUROPE|ly final courts cajole furiously final excuse +4|MIDDLE EAST|uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl diff --git a/framework/resources/Datasources/text_storage/v3_text_reader/region_partitioned/region_1.tbl b/framework/resources/Datasources/text_storage/v3_text_reader/region_partitioned/region_1.tbl new file mode 100644 index 000000000..17441bf96 --- /dev/null +++ b/framework/resources/Datasources/text_storage/v3_text_reader/region_partitioned/region_1.tbl @@ -0,0 +1,5 @@ +0|AFRICA|lar deposits. blithely final packages cajole. regular waters are final requests. regular accounts are according to +1|AMERICA|hs use ironic, even requests. s +2|ASIA|ges. thinly even pinto beans ca +3|EUROPE|ly final courts cajole furiously final excuse +4|MIDDLE EAST|uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl diff --git a/framework/resources/Functional/text_storage/testcases/3wayjoin_DRILL-1421.q b/framework/resources/Functional/text_storage/testcases/3wayjoin_DRILL-1421.sql similarity index 100% rename from framework/resources/Functional/text_storage/testcases/3wayjoin_DRILL-1421.q rename to framework/resources/Functional/text_storage/testcases/3wayjoin_DRILL-1421.sql diff --git a/framework/resources/Functional/text_storage/testcases/DRILL_4814_table_function.e_tsv b/framework/resources/Functional/text_storage/testcases/DRILL_4814_table_function.e_tsv new file mode 100644 index 000000000..ef0b82b62 --- /dev/null +++ b/framework/resources/Functional/text_storage/testcases/DRILL_4814_table_function.e_tsv @@ -0,0 +1,5 @@ +0 AFRICA lar deposits. blithely final packages cajole. regular waters are final requests. regular accounts are according to + +2 ASIA ges. thinly even pinto beans ca +3 EUROPE +4 MIDDLE EAST \ No newline at end of file diff --git a/framework/resources/Functional/text_storage/testcases/DRILL_4814_table_function.sql b/framework/resources/Functional/text_storage/testcases/DRILL_4814_table_function.sql new file mode 100644 index 000000000..d7722d99b --- /dev/null +++ b/framework/resources/Functional/text_storage/testcases/DRILL_4814_table_function.sql @@ -0,0 +1,3 @@ +set `exec.storage.enable_v3_text_reader` = false; +select * from table(`/drill/testdata/text_storage/v3_text_reader/region_malformed.csvh`(type=>'text',lineDelimiter=>'\n',fieldDelimiter=>',',extractHeader=>true)); +reset `exec.storage.enable_v3_text_reader`; \ No newline at end of file diff --git a/framework/resources/Functional/text_storage/testcases/DRILL_5487_truncated_last_row.e_tsv b/framework/resources/Functional/text_storage/testcases/DRILL_5487_truncated_last_row.e_tsv new file mode 100644 index 000000000..ef0b82b62 --- /dev/null +++ b/framework/resources/Functional/text_storage/testcases/DRILL_5487_truncated_last_row.e_tsv @@ -0,0 +1,5 @@ +0 AFRICA lar deposits. blithely final packages cajole. regular waters are final requests. regular accounts are according to + +2 ASIA ges. thinly even pinto beans ca +3 EUROPE +4 MIDDLE EAST \ No newline at end of file diff --git a/framework/resources/Functional/text_storage/testcases/DRILL_5487_truncated_last_row.sql b/framework/resources/Functional/text_storage/testcases/DRILL_5487_truncated_last_row.sql new file mode 100644 index 000000000..00896113b --- /dev/null +++ b/framework/resources/Functional/text_storage/testcases/DRILL_5487_truncated_last_row.sql @@ -0,0 +1,3 @@ +set `exec.storage.enable_v3_text_reader` = true; +select * from `/drill/testdata/text_storage/v3_text_reader/region_malformed.csvh`; +reset `exec.storage.enable_v3_text_reader`; \ No newline at end of file diff --git a/framework/resources/Functional/text_storage/testcases/DRILL_6952_enable_v3_text_reader_default_value.e_tsv b/framework/resources/Functional/text_storage/testcases/DRILL_6952_enable_v3_text_reader_default_value.e_tsv new file mode 100644 index 000000000..9ea724d93 --- /dev/null +++ b/framework/resources/Functional/text_storage/testcases/DRILL_6952_enable_v3_text_reader_default_value.e_tsv @@ -0,0 +1 @@ +exec.storage.enable_v3_text_reader false \ No newline at end of file diff --git a/framework/resources/Functional/text_storage/testcases/DRILL_6952_enable_v3_text_reader_default_value.sql b/framework/resources/Functional/text_storage/testcases/DRILL_6952_enable_v3_text_reader_default_value.sql new file mode 100644 index 000000000..647b5744b --- /dev/null +++ b/framework/resources/Functional/text_storage/testcases/DRILL_6952_enable_v3_text_reader_default_value.sql @@ -0,0 +1 @@ +set `exec.storage.enable_v3_text_reader`; \ No newline at end of file diff --git a/framework/resources/Functional/text_storage/testcases/DRILL_6952_implicit_columns.e_tsv b/framework/resources/Functional/text_storage/testcases/DRILL_6952_implicit_columns.e_tsv new file mode 100644 index 000000000..5161252e7 --- /dev/null +++ b/framework/resources/Functional/text_storage/testcases/DRILL_6952_implicit_columns.e_tsv @@ -0,0 +1,10 @@ +["0","AFRICA","lar deposits. blithely final packages cajole. regular waters are final requests. regular accounts are according to "] null null VARCHAR VARCHAR +["1","AMERICA","hs use ironic, even requests. s"] null null VARCHAR VARCHAR +["2","ASIA","ges. thinly even pinto beans ca"] null null VARCHAR VARCHAR +["3","EUROPE","ly final courts cajole furiously final excuse"] null null VARCHAR VARCHAR +["4","MIDDLE EAST","uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl"] null null VARCHAR VARCHAR +["0","AFRICA","lar deposits. blithely final packages cajole. regular waters are final requests. regular accounts are according to "] nested null VARCHAR VARCHAR +["1","AMERICA","hs use ironic, even requests. s"] nested null VARCHAR VARCHAR +["2","ASIA","ges. thinly even pinto beans ca"] nested null VARCHAR VARCHAR +["3","EUROPE","ly final courts cajole furiously final excuse"] nested null VARCHAR VARCHAR +["4","MIDDLE EAST","uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl"] nested null VARCHAR VARCHAR \ No newline at end of file diff --git a/framework/resources/Functional/text_storage/testcases/DRILL_6952_implicit_columns.sql b/framework/resources/Functional/text_storage/testcases/DRILL_6952_implicit_columns.sql new file mode 100644 index 000000000..eeb94d3da --- /dev/null +++ b/framework/resources/Functional/text_storage/testcases/DRILL_6952_implicit_columns.sql @@ -0,0 +1,3 @@ +set `exec.storage.min_width` = 2; +select *, drillTypeOf(dir0), drillTypeOf(dir1) from dfs.`/drill/testdata/text_storage/v3_text_reader/region_partitioned`; +reset `exec.storage.min_width`; \ No newline at end of file diff --git a/framework/resources/Functional/text_storage/testcases/DRILL_7034_window function.e_tsv b/framework/resources/Functional/text_storage/testcases/DRILL_7034_window function.e_tsv new file mode 100644 index 000000000..87b4d4e3c --- /dev/null +++ b/framework/resources/Functional/text_storage/testcases/DRILL_7034_window function.e_tsv @@ -0,0 +1,5 @@ + +0 +2 +3 +4 \ No newline at end of file diff --git a/framework/resources/Functional/text_storage/testcases/DRILL_7034_window function.sql b/framework/resources/Functional/text_storage/testcases/DRILL_7034_window function.sql new file mode 100644 index 000000000..d02e6a4df --- /dev/null +++ b/framework/resources/Functional/text_storage/testcases/DRILL_7034_window function.sql @@ -0,0 +1 @@ +select max(id) over(order by id) from `/drill/testdata/text_storage/v3_text_reader/region_malformed.csvh`; \ No newline at end of file diff --git a/framework/resources/Functional/text_storage/testcases/drill1831.q b/framework/resources/Functional/text_storage/testcases/drill1831.sql similarity index 100% rename from framework/resources/Functional/text_storage/testcases/drill1831.q rename to framework/resources/Functional/text_storage/testcases/drill1831.sql diff --git a/framework/resources/Functional/text_storage/testcases/drill2739.q b/framework/resources/Functional/text_storage/testcases/drill2739.sql similarity index 100% rename from framework/resources/Functional/text_storage/testcases/drill2739.q rename to framework/resources/Functional/text_storage/testcases/drill2739.sql diff --git a/framework/resources/Functional/text_storage/testcases/drill3178_1.q b/framework/resources/Functional/text_storage/testcases/drill3178_1.sql similarity index 100% rename from framework/resources/Functional/text_storage/testcases/drill3178_1.q rename to framework/resources/Functional/text_storage/testcases/drill3178_1.sql diff --git a/framework/resources/Functional/text_storage/testcases/drill3178_2.q b/framework/resources/Functional/text_storage/testcases/drill3178_2.sql similarity index 100% rename from framework/resources/Functional/text_storage/testcases/drill3178_2.q rename to framework/resources/Functional/text_storage/testcases/drill3178_2.sql diff --git a/framework/resources/Functional/text_storage/testcases/drill3178_3.q b/framework/resources/Functional/text_storage/testcases/drill3178_3.sql similarity index 100% rename from framework/resources/Functional/text_storage/testcases/drill3178_3.q rename to framework/resources/Functional/text_storage/testcases/drill3178_3.sql diff --git a/framework/resources/Functional/text_storage/testcases/drill4484_1.q b/framework/resources/Functional/text_storage/testcases/drill4484_1.sql similarity index 100% rename from framework/resources/Functional/text_storage/testcases/drill4484_1.q rename to framework/resources/Functional/text_storage/testcases/drill4484_1.sql diff --git a/framework/resources/Functional/text_storage/testcases/drill4484_2.q b/framework/resources/Functional/text_storage/testcases/drill4484_2.sql similarity index 100% rename from framework/resources/Functional/text_storage/testcases/drill4484_2.q rename to framework/resources/Functional/text_storage/testcases/drill4484_2.sql diff --git a/framework/resources/Functional/text_storage/testcases/drill4484_3.q b/framework/resources/Functional/text_storage/testcases/drill4484_3.sql similarity index 100% rename from framework/resources/Functional/text_storage/testcases/drill4484_3.q rename to framework/resources/Functional/text_storage/testcases/drill4484_3.sql diff --git a/framework/resources/Functional/text_storage/testcases/drill4484_4.q b/framework/resources/Functional/text_storage/testcases/drill4484_4.sql similarity index 100% rename from framework/resources/Functional/text_storage/testcases/drill4484_4.q rename to framework/resources/Functional/text_storage/testcases/drill4484_4.sql diff --git a/framework/resources/Functional/text_storage/testcases/drill4919_1.q b/framework/resources/Functional/text_storage/testcases/drill4919_1.sql similarity index 100% rename from framework/resources/Functional/text_storage/testcases/drill4919_1.q rename to framework/resources/Functional/text_storage/testcases/drill4919_1.sql diff --git a/framework/resources/Functional/text_storage/testcases/drill4919_2.q b/framework/resources/Functional/text_storage/testcases/drill4919_2.sql similarity index 100% rename from framework/resources/Functional/text_storage/testcases/drill4919_2.q rename to framework/resources/Functional/text_storage/testcases/drill4919_2.sql diff --git a/framework/resources/Functional/text_storage/testcases/drill4919_3.q b/framework/resources/Functional/text_storage/testcases/drill4919_3.sql similarity index 100% rename from framework/resources/Functional/text_storage/testcases/drill4919_3.q rename to framework/resources/Functional/text_storage/testcases/drill4919_3.sql diff --git a/framework/resources/Functional/text_storage/testcases/drill4919_4.q b/framework/resources/Functional/text_storage/testcases/drill4919_4.sql similarity index 100% rename from framework/resources/Functional/text_storage/testcases/drill4919_4.q rename to framework/resources/Functional/text_storage/testcases/drill4919_4.sql diff --git a/framework/resources/Functional/text_storage/testcases/drill4919_5.q b/framework/resources/Functional/text_storage/testcases/drill4919_5.sql similarity index 100% rename from framework/resources/Functional/text_storage/testcases/drill4919_5.q rename to framework/resources/Functional/text_storage/testcases/drill4919_5.sql diff --git a/framework/resources/Functional/text_storage/testcases/drill4919_6.q b/framework/resources/Functional/text_storage/testcases/drill4919_6.sql similarity index 100% rename from framework/resources/Functional/text_storage/testcases/drill4919_6.q rename to framework/resources/Functional/text_storage/testcases/drill4919_6.sql diff --git a/framework/resources/Functional/text_storage/testcases/drill4919_7.q b/framework/resources/Functional/text_storage/testcases/drill4919_7.sql similarity index 100% rename from framework/resources/Functional/text_storage/testcases/drill4919_7.q rename to framework/resources/Functional/text_storage/testcases/drill4919_7.sql diff --git a/framework/resources/Functional/text_storage/testcases/drill950.q b/framework/resources/Functional/text_storage/testcases/drill950.sql similarity index 100% rename from framework/resources/Functional/text_storage/testcases/drill950.q rename to framework/resources/Functional/text_storage/testcases/drill950.sql diff --git a/framework/resources/Functional/text_storage/testcases/select.q b/framework/resources/Functional/text_storage/testcases/select.sql similarity index 100% rename from framework/resources/Functional/text_storage/testcases/select.q rename to framework/resources/Functional/text_storage/testcases/select.sql diff --git a/framework/resources/Functional/text_storage/testcases/select_tsv.q b/framework/resources/Functional/text_storage/testcases/select_tsv.sql similarity index 100% rename from framework/resources/Functional/text_storage/testcases/select_tsv.q rename to framework/resources/Functional/text_storage/testcases/select_tsv.sql diff --git a/framework/resources/Functional/text_storage/testcases/selectbz2.q b/framework/resources/Functional/text_storage/testcases/selectbz2.sql similarity index 100% rename from framework/resources/Functional/text_storage/testcases/selectbz2.q rename to framework/resources/Functional/text_storage/testcases/selectbz2.sql diff --git a/framework/resources/Functional/text_storage/testcases/selectdir.q b/framework/resources/Functional/text_storage/testcases/selectdir.sql similarity index 100% rename from framework/resources/Functional/text_storage/testcases/selectdir.q rename to framework/resources/Functional/text_storage/testcases/selectdir.sql diff --git a/framework/resources/Functional/text_storage/testcases/selectgz.q b/framework/resources/Functional/text_storage/testcases/selectgz.sql similarity index 100% rename from framework/resources/Functional/text_storage/testcases/selectgz.q rename to framework/resources/Functional/text_storage/testcases/selectgz.sql diff --git a/framework/resources/Functional/text_storage/testcases/selectpart.q b/framework/resources/Functional/text_storage/testcases/selectpart.sql similarity index 100% rename from framework/resources/Functional/text_storage/testcases/selectpart.q rename to framework/resources/Functional/text_storage/testcases/selectpart.sql diff --git a/framework/resources/Functional/text_storage/testcases/selectsingledir.q b/framework/resources/Functional/text_storage/testcases/selectsingledir.sql similarity index 100% rename from framework/resources/Functional/text_storage/testcases/selectsingledir.q rename to framework/resources/Functional/text_storage/testcases/selectsingledir.sql diff --git a/framework/resources/Functional/text_storage/testcases/textReadGroup.json b/framework/resources/Functional/text_storage/testcases/textReadGroup.json index 9b837fde1..3d1eab71f 100644 --- a/framework/resources/Functional/text_storage/testcases/textReadGroup.json +++ b/framework/resources/Functional/text_storage/testcases/textReadGroup.json @@ -1,31 +1,31 @@ { - "testId": "textRead", - "type": "group", - "description": "Test text read queries", - "categories": [ - "functional" - ], - "matrices": [ - { - "query-file": ".*.q", - "schema": "dfs", - "output-format": "tsv", - "expected-file": ".*.e_tsv", - "verification-type": [ - "in-memory" - ] - } - ], - "datasources": [ - { - "mode": "cp", - "src": "Datasources/text_storage", - "dest": "text_storage" - }, - { - "mode": "gen", - "src": "Datasources/text_storage/drill-4484.sh", - "dest": "" - } - ] + "testId": "textRead", + "type": "group", + "description": "Test text read queries", + "categories": [ + "functional" + ], + "matrices": [ + { + "query-file": ".*.sql", + "schema": "dfs", + "output-format": "tsv", + "expected-file": ".*.e_tsv", + "verification-type": [ + "in-memory" + ] + } + ], + "datasources": [ + { + "mode": "cp", + "src": "Datasources/text_storage", + "dest": "text_storage" + }, + { + "mode": "gen", + "src": "Datasources/text_storage/drill-4484.sh", + "dest": "" + } + ] }