Adding shenyu-common into the openctest framework

core/add_project.sh

@@ -48,6 +48,16 @@ function setup_alluxio() {
     mvn clean install -DskipTests -Dcheckstyle.skip -Dlicense.skip -Dfindbugs.skip -Dmaven.javadoc.skip=true
 }
 
+function setup_shenyu_common() {
+    # /home/kejiawu2/project/openctest/core/identify_param/app/ctest-shenyu/shenyu/shenyu-common/
+    [ ! -d "app/ctest-shenyu" ] && git clone https://github.com/KobeNorris/shenyu.git app/ctest-shenyu
+    cd app/ctest-shenyu
+    git fetch && git checkout ctest-interception
+    home_dir=$PWD
+    cd $home_dir/shenyu-common/
+    mvn package -DskipTests
+}
+
 function usage() {
     echo "Usage: add_project.sh <main project>"
     exit 1
@@ -64,7 +74,8 @@ function main() {
             hbase) setup_hbase ;;
             zookeeper) setup_zookeeper ;;
             alluxio) setup_alluxio ;;
-            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper and alluxio." ;;
+            shenyu-common) setup_shenyu_common ;;
+            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper, alluxio and shenyu-common." ;;
         esac
     fi
 }

core/ctest_const.py

@@ -12,18 +12,21 @@
 HBASE = "hbase-server"
 ZOOKEEPER = "zookeeper-server"
 ALLUXIO = "alluxio-core"
+SYCOMMON = "shenyu-common"
 
 CTEST_HADOOP_DIR = os.path.join(APP_DIR, "ctest-hadoop")
 CTEST_HBASE_DIR = os.path.join(APP_DIR, "ctest-hbase")
 CTEST_ZK_DIR = os.path.join(APP_DIR, "ctest-zookeeper")
 CTEST_ALLUXIO_DIR = os.path.join(APP_DIR, "ctest-alluxio")
+CTEST_SYCOMMON_DIR = os.path.join(APP_DIR, "ctest-shenyu")
 
 PROJECT_DIR = {
     HCOMMON: CTEST_HADOOP_DIR,
     HDFS: CTEST_HADOOP_DIR,
     HBASE: CTEST_HBASE_DIR,
     ZOOKEEPER: CTEST_ZK_DIR,
     ALLUXIO: CTEST_ALLUXIO_DIR,
+    SYCOMMON: CTEST_SYCOMMON_DIR,
 }
 
 
@@ -34,6 +37,7 @@
     HBASE: "hbase-server",
     ZOOKEEPER: "zookeeper-server",
     ALLUXIO: "core",
+    SYCOMMON: "shenyu-common",
 }
 
 
@@ -58,6 +62,7 @@
         os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/worker", SUREFIRE_SUBDIR),
         os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/master", SUREFIRE_SUBDIR),
     ],
+    SYCOMMON: [os.path.join(CTEST_SYCOMMON_DIR, MODULE_SUBDIR[SYCOMMON], SUREFIRE_SUBDIR)],
 }
 
 # default or deprecate conf path
@@ -74,7 +79,8 @@
     HDFS: os.path.join(DEFAULT_CONF_DIR, HDFS + "-default.tsv"),
     HBASE: os.path.join(DEFAULT_CONF_DIR, HBASE + "-default.tsv"),
     ALLUXIO: os.path.join(DEFAULT_CONF_DIR, ALLUXIO + "-default.tsv"),
-    ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv")
+    ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv"),
+    SYCOMMON: os.path.join(DEFAULT_CONF_DIR, SYCOMMON + "-default.tsv")
 }
 
 
@@ -96,7 +102,12 @@
     ],
     ALLUXIO: [
         os.path.join(CTEST_ALLUXIO_DIR, "core/alluxio-ctest.properties")
-    ]
+    ],
+    SYCOMMON: [
+        # /home/kejiawu2/project/openctest/core/app/ctest-shenyu/shenyu-common/src/main/java/org/apache/shenyu/common/config/InjectionValuePair.txt
+        # /home/kejiawu2/project/openctest/core/app/ctest-shenyu/shenyu-common/src/main/java/org/apache/shenyu/common/config/InjectionValuePair.txt
+        os.path.join(CTEST_SYCOMMON_DIR, "shenyu-common/src/main/java/org/apache/shenyu/common/config/InjectionValuePair.txt")
+    ],
 }
 
 

core/default_configs/hbase-server-default.tsv

@@ -1,4 +1,4 @@
-hbase.rootdir	${hbase.tmp.dir}/hbase	The directory shared by region servers and into     which HBase persists.  The URL should be 'fully-qualified'     to include the filesystem scheme.  For example, to specify the     HDFS directory '/hbase' where the HDFS instance's namenode is     running at namenode.example.org on port 9000, set this value to:     hdfs://namenode.example.org:9000/hbase.  By default, we write     to whatever ${hbase.tmp.dir} is set too -- usually /tmp --     so change this configuration or else all data will be lost on     machine restart.
+hbase.rootdir   ${hbase.tmp.dir}/hbase	The directory shared by region servers and into     which HBase persists.  The URL should be 'fully-qualified'     to include the filesystem scheme.  For example, to specify the     HDFS directory '/hbase' where the HDFS instance's namenode is     running at namenode.example.org on port 9000, set this value to:     hdfs://namenode.example.org:9000/hbase.  By default, we write     to whatever ${hbase.tmp.dir} is set too -- usually /tmp --     so change this configuration or else all data will be lost on     machine restart.
 zookeeper.recovery.retry.maxsleeptime	60000	Max sleep time before retry zookeeper operations in milliseconds,     a max time is needed here so that sleep time won't grow unboundedly
 hbase.master.port	16000	The port the HBase Master should bind to.
 hbase.master.info.bindAddress	0.0.0.0	The bind address for the HBase Master web UI
@@ -74,7 +74,7 @@ hbase.status.multicast.address.port	16100	Multicast port to use for the status p
 hbase.dynamic.jars.dir	${hbase.rootdir}/lib	The directory from which the custom filter JARs can be loaded       dynamically by the region server without the need to restart. However,       an already loaded filter/co-processor class would not be un-loaded. See       HBASE-1936 for more details.        Does not apply to coprocessors.
 hbase.master.loadbalancer.class	org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer	Class used to execute the regions balancing when the period occurs.       See the class comment for more on how it works       http://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html       It replaces the DefaultLoadBalancer as the default (since renamed       as the SimpleLoadBalancer).
 hbase.master.normalizer.class	org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer	Class used to execute the region normalization when the period occurs.       See the class comment for more on how it works       http://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.html
-hbase.procedure.master.classes		A comma-separated list of     org.apache.hadoop.hbase.procedure.MasterProcedureManager procedure managers that are     loaded by default on the active HMaster process. A procedure is identified by its signature and     users can use the signature and an instant name to trigger an execution of a globally barriered     procedure. After implementing your own MasterProcedureManager, just put it in HBase's classpath     and add the fully qualified class name here.
+hbase.procedure.master.classes      A comma-separated list of     org.apache.hadoop.hbase.procedure.MasterProcedureManager procedure managers that are     loaded by default on the active HMaster process. A procedure is identified by its signature and     users can use the signature and an instant name to trigger an execution of a globally barriered     procedure. After implementing your own MasterProcedureManager, just put it in HBase's classpath     and add the fully qualified class name here.
 hbase.regionserver.storefile.refresh.period	0	The period (in milliseconds) for refreshing the store files for the secondary regions. 0       means this feature is disabled. Secondary regions sees new files (from flushes and       compactions) from primary once the secondary region refreshes the list of files in the       region (there is no notification mechanism). But too frequent refreshes might cause       extra Namenode pressure. If the files cannot be refreshed for longer than HFile TTL       (hbase.master.hfilecleaner.ttl) the requests are rejected. Configuring HFile TTL to a larger       value is also recommended with this setting.
 hbase.http.filter.initializers	org.apache.hadoop.hbase.http.lib.StaticUserWebFilter	A comma separated list of class names. Each class in the list must extend       org.apache.hadoop.hbase.http.FilterInitializer. The corresponding Filter will       be initialized. Then, the Filter will be applied to all user facing jsp       and servlet web pages.       The ordering of the list defines the ordering of the filters.       The default StaticUserWebFilter add a user principal as defined by the       hbase.http.staticuser.user property.
 hbase.http.max.threads	16	The maximum number of threads that the HTTP Server will create in its       ThreadPool.

core/default_configs/shenyu-common-default.tsv

@@ -0,0 +1,38 @@
+Scheduler.enabled	false	.
+Scheduler.type	fixed	.
+ExtPlugin.enabled	false	.
+ExtPlugin.threads	1	.
+ExtPlugin.scheduleTime	300	.
+ExtPlugin.scheduleDelay	30	.
+MatchCache.enabled	false	.
+MatchCache.maxFreeMemory	256	.
+ExcludePath.enabled	false	.
+FallbackPath.enabled	false	.
+Health.enabled	false	.
+FileConfig.enabled	false	.
+FileConfig.maxSize	10	.
+UpstreamCheck.SwitchConfig.local	true	.
+UpstreamCheck.enabled	false	.
+UpstreamCheck.timeout	3000	.
+UpstreamCheck.healthyThreshold	1	.
+UpstreamCheck.unhealthyThreshold	1	.
+UpstreamCheck.interval	5000	.
+UpstreamCheck.printEnabled	false	.
+UpstreamCheck.printInterval	60000	.
+CrossFilterConfig.enabled	false	.
+CrossFilterConfig.allowedMethods	*	.
+CrossFilterConfig.allowedAnyOrigin	fales	.
+CrossFilterConfig.maxAge	18000	.
+CrossFilterConfig.allowCredentials	false	.
+CrossFilterConfig.AllowedOriginConfig.spacer	.	.
+InstanceConfig.enabled	false	.
+RibbonConfig.serverListRefreshInterval	10000	.
+Local.enabled	false	.
+WebsocketConfig.maxFramePayloadSize	10	.
+SharedPool.enable	false	.
+SharedPool.prefix	shenyu-shared	.
+SharedPool.corePoolSize	200	.
+SharedPool.maximumPoolSize	Integer.MAX_VALUE	.
+SharedPool.keepAliveTime	60000L	.
+SharedPool.maxWorkQueueMemory	MemoryLimitCalculator.defaultLimit()	.
+MetricsConfig.enabled	false	.

core/generate_ctest/ctest_mapping/ctests-shenyu-common.json

@@ -0,0 +1 @@
+{"ExtPlugin.scheduleTime": ["org.apache.shenyu.common.config.ShenyuConfigTest#testExtPlugin"], "FileConfig.maxSize": ["org.apache.shenyu.common.config.ShenyuConfigTest#testFileConfig"]}

core/generate_ctest/inject.py

@@ -14,7 +14,7 @@ def inject_config(param_value_pairs):
     for p, v in param_value_pairs.items():
         print(">>>>[ctest_core] injecting {} with value {}".format(p, v))
 
-    if project in [ZOOKEEPER, ALLUXIO]:
+    if project in [ZOOKEEPER, ALLUXIO, SYCOMMON]:
         for inject_path in INJECTION_PATH[project]:
             print(">>>>[ctest_core] injecting into file: {}".format(inject_path))
             file = open(inject_path, "w")
@@ -41,7 +41,7 @@ def inject_config(param_value_pairs):
 
 def clean_conf_file(project):
     print(">>>> cleaning injected configuration from file")
-    if project in [ZOOKEEPER, ALLUXIO]:
+    if project in [ZOOKEEPER, ALLUXIO, SYCOMMON]:
         for inject_path in INJECTION_PATH[project]:
             file = open(inject_path, "w")
             file.write("\n")

core/generate_ctest/main.py

@@ -33,7 +33,7 @@ def test_value_pair(test_input):
         mt_file = open(os.path.join(GENCTEST_TR_DIR, project, MT_FILE.format(id=param)), "w")
 
         associated_tests = mapping[param] if param in mapping else []
-        if len(mapping[param]) != 0:
+        if param in mapping.keys() and len(mapping[param]) != 0:
             for value in values:
                 tr = run_test_seperate(param, value, associated_tests)
 

core/generate_ctest/program_input.py

@@ -4,11 +4,11 @@
     # run mode
     "run_mode": "generate_ctest", # string
     # name of the project, i.e. hadoop-common, hadoop-hdfs, see constant.py
-    "project": "hadoop-common", # string
+    "project": "shenyu-common", # string
     # path to param -> tests json mapping
-    "mapping_path": "../../data/ctest_mapping/opensource-hadoop-common.json", # string
+    "mapping_path": "../../data/ctest_mapping/opensource-shenyu-common.json", # string
     # good values of params tests will be run against
-    "param_value_tsv": "sample-hadoop-common.tsv", # string
+    "param_value_tsv": "sample-shenyu-common.tsv", # string
     # display the terminal output live, without saving any results
     "display_mode": False, # bool
     # whether to use mvn test or mvn surefire:test

core/generate_ctest/sample-shenyu-common.tsv

@@ -0,0 +1,38 @@
+Scheduler.enabled	true SKIP
+Scheduler.type	SKIP SKIP
+ExtPlugin.enabled	true SKIP
+ExtPlugin.threads	0	2
+ExtPlugin.scheduleTime	150	600
+ExtPlugin.scheduleDelay	15	60
+MatchCache.enabled	true SKIP
+MatchCache.maxFreeMemory	128	512
+ExcludePath.enabled	true SKIP
+FallbackPath.enabled	true SKIP
+Health.enabled	true SKIP
+FileConfig.enabled	true SKIP
+FileConfig.maxSize	1	20
+UpstreamCheck.SwitchConfig.local	false SKIP
+UpstreamCheck.enabled	true SKIP
+UpstreamCheck.timeout	1500	6000
+UpstreamCheck.healthyThreshold	0	2
+UpstreamCheck.unhealthyThreshold	0	2
+UpstreamCheck.interval	2500	10000
+UpstreamCheck.printEnabled	true SKIP
+UpstreamCheck.printInterval	30000	120000
+CrossFilterConfig.enabled	true SKIP
+CrossFilterConfig.allowedMethods	SKIP SKIP
+CrossFilterConfig.allowedAnyOrigin	SKIP SKIP
+CrossFilterConfig.maxAge	9000	36000
+CrossFilterConfig.allowCredentials	true SKIP
+CrossFilterConfig.AllowedOriginConfig.spacer	SKIP SKIP
+InstanceConfig.enabled	true SKIP
+RibbonConfig.serverListRefreshInterval	5000	20000
+Local.enabled	true SKIP
+WebsocketConfig.maxFramePayloadSize	1	20
+SharedPool.enable	true SKIP
+SharedPool.prefix	SKIP SKIP
+SharedPool.corePoolSize	100	400
+SharedPool.maximumPoolSize	SKIP SKIP
+SharedPool.keepAliveTime	SKIP SKIP
+SharedPool.maxWorkQueueMemory	SKIP SKIP
+MetricsConfig.enabled	true SKIP

core/generate_value/value_generation.py

@@ -21,10 +21,14 @@ def read_tsv(module):
     tsv_file = open("../default_configs/" + module + "-default.tsv")
     read_tsv = csv.reader(tsv_file, delimiter="\t")
     for row in read_tsv:
+        print(row)
         params.append(Param(row[0], row[1], row[2]))
     if module == "zookeeper-server":
         assert len(params) == 32
         return 32
+    elif module == "shenyu-common":
+        assert len(params) == 38
+        return 38
     else:
         assert len(params) == 90
         return 90
@@ -105,6 +109,8 @@ def print_params(module):
     f = open(module + output, "w")
     if module == "zookeeper-server":
         assert len(params) == 32
+    elif module == "shenyu-common":
+        assert len(params) == 38
     else:
         assert len(params) >= 90
     for param in params:
@@ -117,15 +123,15 @@ def print_params(module):
                 print(param.dvalue)
                 print(param.description)
                 print("----------------------")
-            f.write("SKIP\tSKIP\n")
+            f.write("SKIP SKIP\n")
             unhandled.append(param)
         else:
             pcnt += 1
             tmp_cnt += 1 # for the default value
             tmp_cnt += len(param.gvalues)
             assert len(param.gvalues) <= 2
             if len(param.gvalues) == 1:
-                f.write(str(param.gvalues[0]) + "\tSKIP\n")
+                f.write(str(param.gvalues[0]) + " SKIP\n")
             elif len(param.gvalues) == 2:
                 f.write(str(param.gvalues[0]) + "\t" + str(param.gvalues[1]) + "\n")
             else:

core/identify_param/add_project.sh

@@ -47,6 +47,16 @@ function setup_alluxio() {
     mvn clean install -DskipTests -Dcheckstyle.skip -Dlicense.skip -Dfindbugs.skip -Dmaven.javadoc.skip=true
 }
 
+function setup_shenyu_common() {
+    # /home/kejiawu2/project/openctest/core/identify_param/app/ctest-shenyu/shenyu/shenyu-common/
+    [ ! -d "app/ctest-shenyu" ] && git clone https://github.com/KobeNorris/shenyu.git app/ctest-shenyu
+    cd app/ctest-shenyu
+    git fetch && git checkout ctest-logging
+    home_dir=$PWD
+    cd $home_dir/shenyu-common/
+    mvn package -DskipTests
+}
+
 function usage() {
     echo "Usage: add_project.sh <main project>"
     exit 1
@@ -63,7 +73,8 @@ function main() {
             hbase) setup_hbase ;;
             zookeeper) setup_zookeeper ;;
             alluxio) setup_alluxio ;;
-            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper and alluxio." ;;
+            shenyu-common) setup_shenyu_common ;;
+            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper, alluxio and shenyu-common." ;;
         esac
     fi
 }

core/identify_param/constant.py

@@ -7,20 +7,23 @@
 CTEST_HBASE_DIR = os.path.join(APP_DIR, "ctest-hbase")
 CTEST_ZOOKEEPER_DIR = os.path.join(APP_DIR, "ctest-zookeeper")
 CTEST_ALLUXIO_DIR = os.path.join(APP_DIR, "ctest-alluxio")
+CTEST_SHENYU_DIR = os.path.join(APP_DIR, "ctest-shenyu")
 
 MODULE_PATH = {
     "hadoop-common": CTEST_HADOOP_DIR,
     "hadoop-hdfs": CTEST_HADOOP_DIR,
     "hbase-server": CTEST_HBASE_DIR,
-    "alluxio-core": CTEST_ALLUXIO_DIR
+    "alluxio-core": CTEST_ALLUXIO_DIR,
+    "shenyu-common": CTEST_SHENYU_DIR
 }
 
 SRC_SUBDIR = {
     "hadoop-common": "hadoop-common-project/hadoop-common",
     "hadoop-hdfs": "hadoop-hdfs-project/hadoop-hdfs",
     "hbase-server": "hbase-server",
     "zookeeper-server": "zookeeper-server",
-    "alluxio-core": "core"
+    "alluxio-core": "core",
+    "shenyu-common": "shenyu-common"
 }
 
 MVN_TEST_PATH = {
@@ -29,14 +32,16 @@
     "hbase-server": os.path.join(CTEST_HBASE_DIR, SRC_SUBDIR["hbase-server"]),
     "zookeeper-server": os.path.join(CTEST_ZOOKEEPER_DIR, SRC_SUBDIR["zookeeper-server"]),
     "alluxio-core": os.path.join(CTEST_ALLUXIO_DIR, SRC_SUBDIR["alluxio-core"]),
+    "shenyu-common": os.path.join(CTEST_SHENYU_DIR, SRC_SUBDIR["shenyu-common"])
 }
 
 LOCAL_CONF_PATH = {
     "hadoop-common": "results/hadoop-common/conf_params.txt",
     "hadoop-hdfs": "results/hadoop-hdfs/conf_params.txt",
     "hbase-server": "results/hbase-server/conf_params.txt",
     "zookeeper-server": "results/zookeeper-server/conf_params.txt",
-    "alluxio-core": "results/alluxio-core/conf_params.txt"
+    "alluxio-core": "results/alluxio-core/conf_params.txt",
+    "shenyu-common": "results/shenyu-common/conf_params.txt"
 }
 
 SUREFIRE_SUBDIR = "target/surefire-reports/*"
@@ -63,6 +68,9 @@
         os.path.join(CTEST_ALLUXIO_DIR, "core/server/proxy", SUREFIRE_SUBDIR),
         os.path.join(CTEST_ALLUXIO_DIR, "core/server/worker", SUREFIRE_SUBDIR),
         os.path.join(CTEST_ALLUXIO_DIR, "core/server/master", SUREFIRE_SUBDIR)
+    ],
+    "shenyu-common": [
+        os.path.join(CTEST_SHENYU_DIR, "shenyu-common", SUREFIRE_SUBDIR)
     ]
 }
 
@@ -83,5 +91,8 @@
     ],
     "alluxio-core": [
         os.path.join("surefire-reports/alluxio-core", LOCAL_SUREFIRE_SUFFIX)
+    ],
+    "shenyu-common": [
+        os.path.join("surefire-reports/shenyu-common", LOCAL_SUREFIRE_SUFFIX)
     ]
 }

core/identify_param/identify_param.sh

@@ -12,9 +12,9 @@ function main() {
       usage
     else
         case $project in
-            hadoop-common | hadoop-hdfs | hbase-server | zookeeper-server | alluxio-core) python3 runner.py $project; python3 collector.py $project ;;
+            hadoop-common | hadoop-hdfs | hbase-server | zookeeper-server | alluxio-core | shenyu-common) python3 runner.py $project; python3 collector.py $project ;;
             -h | --help) usage ;;
-            *) echo "Unexpected project: $project - only support hadoop-common, hadoop-hdfs, hbase-server, zookeeper-server and alluxio-core." ;;
+            *) echo "Unexpected project: $project - only support hadoop-common, hadoop-hdfs, hbase-server, zookeeper-server, alluxio-core and shenyu-common." ;;
         esac
     fi
 }