Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding Druid (Druid/core module) to openctest #36

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Adding Druid (Druid/core module) to openctest #36

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file added .DS_Store
View file
Open in desktop
Binary file not shown.
Binary file added core/.DS_Store
View file
Open in desktop
Binary file not shown.
11 changes: 10 additions & 1 deletion core/add_project.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,14 @@ function setup_alluxio() {
mvn clean install -DskipTests -Dcheckstyle.skip -Dlicense.skip -Dfindbugs.skip -Dmaven.javadoc.skip=true
}

function setup_druid() {
[ ! -d "app/ctest-druid" ] && git clone https://github.com/b06902047/druid.git app/ctest-druid
cd app/ctest-druid
git fetch && git checkout ctest-core-inject-release
cd core
mvn install -DskipTests -Dcheckstyle.skip
}

function usage() {
echo "Usage: add_project.sh <main project>"
exit 1
Expand All @@ -64,7 +72,8 @@ function main() {
hbase) setup_hbase ;;
zookeeper) setup_zookeeper ;;
alluxio) setup_alluxio ;;
*) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper and alluxio." ;;
druid) setup_druid ;;
*) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper, alluxio and druid." ;;
esac
fi
}
Expand Down
11 changes: 10 additions & 1 deletion core/ctest_const.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,21 @@
HBASE = "hbase-server"
ZOOKEEPER = "zookeeper-server"
ALLUXIO = "alluxio-core"
DCORE = "druid-core"

CTEST_HADOOP_DIR = os.path.join(APP_DIR, "ctest-hadoop")
CTEST_HBASE_DIR = os.path.join(APP_DIR, "ctest-hbase")
CTEST_ZK_DIR = os.path.join(APP_DIR, "ctest-zookeeper")
CTEST_ALLUXIO_DIR = os.path.join(APP_DIR, "ctest-alluxio")
CTEST_DRUID_DIR = os.path.join(APP_DIR, "ctest-druid")

PROJECT_DIR = {
HCOMMON: CTEST_HADOOP_DIR,
HDFS: CTEST_HADOOP_DIR,
HBASE: CTEST_HBASE_DIR,
ZOOKEEPER: CTEST_ZK_DIR,
ALLUXIO: CTEST_ALLUXIO_DIR,
DCORE: CTEST_DRUID_DIR
}


Expand All @@ -34,6 +37,7 @@
HBASE: "hbase-server",
ZOOKEEPER: "zookeeper-server",
ALLUXIO: "core",
DCORE: "core"
}


Expand All @@ -58,6 +62,7 @@
os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/worker", SUREFIRE_SUBDIR),
os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/master", SUREFIRE_SUBDIR),
],
DCORE: [os.path.join(CTEST_DRUID_DIR, MODULE_SUBDIR[DCORE], SUREFIRE_SUBDIR)],
}

# default or deprecate conf path
Expand All @@ -74,7 +79,8 @@
HDFS: os.path.join(DEFAULT_CONF_DIR, HDFS + "-default.tsv"),
HBASE: os.path.join(DEFAULT_CONF_DIR, HBASE + "-default.tsv"),
ALLUXIO: os.path.join(DEFAULT_CONF_DIR, ALLUXIO + "-default.tsv"),
ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv")
ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv"),
DCORE: os.path.join(DEFAULT_CONF_DIR, DCORE + "-default.tsv")
}


Expand All @@ -96,6 +102,9 @@
],
ALLUXIO: [
os.path.join(CTEST_ALLUXIO_DIR, "core/alluxio-ctest.properties")
],
DCORE: [
os.path.join(CTEST_DRUID_DIR, "core/core-ctest.xml")
]
}

Expand Down
538 changes: 538 additions & 0 deletions core/default_configs/druid-core-default.tsv
View file
Open in desktop

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions core/generate_ctest/ctest_mapping/ctests-druid-core.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"druid.emitter.http.minHttpTimeoutMillis": ["org.apache.druid.java.util.emitter.core.HttpPostEmitterStressTest#testLargeAndSmallEventsQueueLimit", "org.apache.druid.java.util.emitter.core.HttpPostEmitterStressTest#testLargeEventsQueueLimit", "org.apache.druid.java.util.emitter.core.EmitterTest#testBasicAuthenticationAndNewlineSeparating", "org.apache.druid.java.util.emitter.core.ParametrizedUriEmitterTest#testEmitterWithFeedUriExtractor", "org.apache.druid.java.util.emitter.core.EmitterTest#testSanityWithGeneralizedCreation", "org.apache.druid.java.util.emitter.core.HttpEmitterTest#timeoutEmptyQueue", "org.apache.druid.java.util.emitter.core.EmitterTest#testGzipContentEncoding", "org.apache.druid.java.util.emitter.core.EmitterTest#testFailedEmission", "org.apache.druid.java.util.emitter.core.ParametrizedUriEmitterTest#testEmitterWithParametrizedUriExtractor", "org.apache.druid.java.util.emitter.core.ParametrizedUriEmitterTest#testEmitterWithMultipleFeeds", "org.apache.druid.java.util.emitter.core.HttpPostEmitterStressTest#eventCountBased", "org.apache.druid.java.util.emitter.core.EmitterTest#testSizeBasedEmission", "org.apache.druid.java.util.emitter.core.EmitterTest#testSanity", "org.apache.druid.java.util.emitter.core.EmitterTest#testBatchSplitting"]}
1 change: 1 addition & 0 deletions core/generate_ctest/ctest_mapping/ctests-hadoop-common.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{}
4 changes: 2 additions & 2 deletions core/generate_ctest/inject.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ def inject_config(param_value_pairs):
for p, v in param_value_pairs.items():
print(">>>>[ctest_core] injecting {} with value {}".format(p, v))

if project in [ZOOKEEPER, ALLUXIO]:
if project in [ZOOKEEPER, ALLUXIO, DCORE]:
for inject_path in INJECTION_PATH[project]:
print(">>>>[ctest_core] injecting into file: {}".format(inject_path))
file = open(inject_path, "w")
Expand All @@ -41,7 +41,7 @@ def inject_config(param_value_pairs):

def clean_conf_file(project):
print(">>>> cleaning injected configuration from file")
if project in [ZOOKEEPER, ALLUXIO]:
if project in [ZOOKEEPER, ALLUXIO, DCORE]:
for inject_path in INJECTION_PATH[project]:
file = open(inject_path, "w")
file.write("\n")
Expand Down
2 changes: 1 addition & 1 deletion core/generate_ctest/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def test_value_pair(test_input):
mt_file = open(os.path.join(GENCTEST_TR_DIR, project, MT_FILE.format(id=param)), "w")

associated_tests = mapping[param] if param in mapping else []
if len(mapping[param]) != 0:
if len(associated_tests) != 0:
for value in values:
tr = run_test_seperate(param, value, associated_tests)

Expand Down
10 changes: 5 additions & 5 deletions core/generate_ctest/program_input.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,17 @@
# run mode
"run_mode": "generate_ctest", # string
# name of the project, i.e. hadoop-common, hadoop-hdfs, see constant.py
"project": "hadoop-common", # string
"project": "druid-core", # string
# path to param -> tests json mapping
"mapping_path": "../../data/ctest_mapping/opensource-hadoop-common.json", # string
"mapping_path": "../../data/ctest_mapping/opensource-druid-core.json", # string
# good values of params tests will be run against
"param_value_tsv": "sample-hadoop-common.tsv", # string
"param_value_tsv": "sample-druid-core.tsv", # string
# display the terminal output live, without saving any results
"display_mode": False, # bool
# whether to use mvn test or mvn surefire:test
"use_surefire": True, # bool
"use_surefire": False, # bool
# additional maven options to pass to `mvn surefire:test -Dtest=...`
"maven_args": [], # list of strings, each element is an option
"maven_args": ["-Dcheckstyle.skip"], # list of strings, each element is an option
# timeout on the mvn test command
"cmd_timeout": None, # int
}
Expand Down
1 change: 1 addition & 0 deletions core/generate_ctest/sample-druid-core.tsv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
druid.emitter.http.minHttpTimeoutMillis 1 -1
2 changes: 2 additions & 0 deletions ...te_ctest/test_result/druid-core/missing_test_druid.emitter.http.minHttpTimeoutMillis.list
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
>>>>[ctest_core] missing test for druid-core druid.emitter.http.minHttpTimeoutMillis 1
>>>>[ctest_core] missing test for druid-core druid.emitter.http.minHttpTimeoutMillis -1
38 changes: 38 additions & 0 deletions ...rate_ctest/test_result/druid-core/test_result_druid.emitter.http.minHttpTimeoutMillis.tsv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.HttpPostEmitterStressTest#testLargeAndSmallEventsQueueLimit 1 p 3.846
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.HttpPostEmitterStressTest#testLargeEventsQueueLimit 1 p 4.651
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.EmitterTest#testBasicAuthenticationAndNewlineSeparating 1 p 1.566
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.ParametrizedUriEmitterTest#testEmitterWithFeedUriExtractor 1 p 1.844
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.EmitterTest#testSanityWithGeneralizedCreation 1 p 1.703
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.HttpEmitterTest#timeoutEmptyQueue 1 p 2.673
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.EmitterTest#testGzipContentEncoding 1 p 1.639
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.EmitterTest#testFailedEmission 1 p 2.605
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.ParametrizedUriEmitterTest#testEmitterWithParametrizedUriExtractor 1 p 1.753
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.ParametrizedUriEmitterTest#testEmitterWithMultipleFeeds 1 p 1.882
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.HttpPostEmitterStressTest#eventCountBased 1 p 12.87
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.HttpEmitterConfigTest#testDefaults 1 f 1.167
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.EmitterTest#testTimeBasedEmission 1 f 1.934
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.EmitterTest#testSizeBasedEmission 1 p 1.693
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.HttpEmitterConfigTest#testSettingEverything 1 f 1.289
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.EmitterTest#testSanity 1 p 1.66
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.HttpEmitterConfigTest#testSettingEverythingLegacy 1 f 1.35
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.HttpEmitterConfigTest#testDefaultsLegacy 1 f 1.625
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.EmitterTest#testBatchSplitting 1 p 1.695
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.EmitterTest#testGzipContentEncoding -1 p 1.768
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.HttpEmitterTest#timeoutEmptyQueue -1 p 2.63
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.ParametrizedUriEmitterTest#testEmitterWithMultipleFeeds -1 p 1.758
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.HttpPostEmitterStressTest#eventCountBased -1 p 4.78
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.HttpPostEmitterStressTest#testLargeAndSmallEventsQueueLimit -1 p 9.668
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.EmitterTest#testFailedEmission -1 p 2.562
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.HttpEmitterConfigTest#testDefaults -1 f 1.391
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.EmitterTest#testSanityWithGeneralizedCreation -1 p 1.726
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.EmitterTest#testBatchSplitting -1 p 2.087
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.EmitterTest#testSanity -1 p 1.609
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.HttpEmitterConfigTest#testDefaultsLegacy -1 f 1.132
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.HttpPostEmitterStressTest#testLargeEventsQueueLimit -1 p 7.816
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.ParametrizedUriEmitterTest#testEmitterWithFeedUriExtractor -1 p 1.725
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.EmitterTest#testTimeBasedEmission -1 f 1.735
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.EmitterTest#testSizeBasedEmission -1 p 1.684
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.HttpEmitterConfigTest#testSettingEverything -1 f 1.081
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.ParametrizedUriEmitterTest#testEmitterWithParametrizedUriExtractor -1 p 2.278
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.HttpEmitterConfigTest#testSettingEverythingLegacy -1 f 1.476
druid.emitter.http.minHttpTimeoutMillis org.apache.druid.java.util.emitter.core.EmitterTest#testBasicAuthenticationAndNewlineSeparating -1 p 2.205
Empty file added core/generate_ctest/test_result/hadoop-common/missing_test_ftp.bytes-per-checksum.list
View file
Open in desktop
Empty file.
Empty file added core/generate_ctest/test_result/hadoop-common/test_result_ftp.bytes-per-checksum.tsv
View file
Open in desktop
Empty file.
Loading