diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 29b1afc..fe78598 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,8 +7,8 @@ on: branches: - main -jobs: - Linux: +jobs: + Linux_cli: name: Linux - Ubuntu Run runs-on: ubuntu-latest steps: @@ -36,9 +36,9 @@ jobs: - name: Run Tests Suits run: | - find test-suites -type f -name "*.yaml" -exec python ./testing.py -f {} -V \; + find test-suites -type f -name "*.yaml" -exec python ./testing.py -f {} -V -e cli \; - Windows: + Windows_cli: name: Windows Run runs-on: windows-latest steps: @@ -74,7 +74,7 @@ jobs: - name: Run Test Suits shell: bash run: | - find test-suites -type f -name "*.yaml" -exec python ./testing.py -f {} -V \; + find test-suites -type f -name "*.yaml" -exec python ./testing.py -f {} -V -e cli \; # MacOS: @@ -100,4 +100,4 @@ jobs: # - name: Run Tests Suits # run: | - # find test-suites -type f -name "*.yaml" -exec python ./testing.py -f {} -V \; + # find test-suites -type f -name "*.yaml" -exec python ./testing.py -f {} -V -e cli \; diff --git a/README.md b/README.md index ab5a5b0..fb480b1 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,10 @@ code-files: - path: random-password-generator-example/app.py test-cases: - name: Check the password is generated in the correct length - command: call getRandomPassword(12) + function-call: getRandomPassword(12) expected-pattern: '\"[\w\W]{12}\"' - name: Check the password is generated in the correct length - command: call getRandomPassword() + function-call: getRandomPassword() expected-pattern: 'missing 1 required positional argument' ``` diff --git a/deploy.sh b/deploy.sh new file mode 100644 index 0000000..000ebe3 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +# Accept the repo path as an argument +repo_path=$1 + +# Run the metacall-deploy command and store the output +output=$(metacall-deploy --inspect OpenAPIv3 --dev --workdir "$repo_path") + +# Parse the JSON output using jq to extract the server URL and paths +server_url=$(echo "$output" | jq -r '.[0].servers[0].url') +paths=$(echo "$output" | jq -r '.[0].paths | keys[]') + +# Output the server URL and paths +echo "Server URL: $server_url" +echo "Available Paths:" +for path in $paths; do + echo "$server_url$path" +done + +# Set the server URL as an environment variable +export SERVER_URL=$server_url + + +# multi line comment +: ' Exmaple output +[ + { + "openapi": "3.0.0", + "info": { + "title": "MetaCall Cloud FaaS deployment 'time-app-web'", + "description": "", + "version": "v1" + }, + "servers": [ + { + "url": "http://localhost:9000/aa759149a70a/time-app-web/v1", + "description": "MetaCall Cloud FaaS" + } + ], + "paths": { + "/call/time": { + "get": { + "summary": "", + "description": "", + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + }, + "/call/index": { + "get": { + "summary": "", + "description": "", + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + } + } + } +] +' \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index be2b74d..217629d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,3 @@ -PyYAML==6.0.1 +config==0.5.1 +metacall==0.5.0 +PyYAML==6.0.2 diff --git a/test-auth-function-mesh.yaml b/test-auth-function-mesh.yaml deleted file mode 100644 index 593e816..0000000 --- a/test-auth-function-mesh.yaml +++ /dev/null @@ -1,19 +0,0 @@ -project: auth-function-mesh -repo-url: https://github.com/metacall/examples -code-files: - - name: auth.py - path: examples/auth-function-mesh/auth.py - test-cases: - - name: Check the encrypt function - command: call encrypt("asd") - expected-pattern: 'eyJhbGciOiJIUzI1NiJ9.YXNk.QNa-p8QpuHcVUDMN_Ih4x4vidWp31365GM4zrSr3t0s' - - name: Check the decrypt function - command: call decrypt("eyJhbGciOiJIUzI1NiJ9.YXNk.QNa-p8QpuHcVUDMN_Ih4x4vidWp31365GM4zrSr3t0s") - expected-pattern: 'asd' - - - - - - - \ No newline at end of file diff --git a/test-auth-middleware.yaml b/test-auth-middleware.yaml deleted file mode 100644 index e2c8095..0000000 --- a/test-auth-middleware.yaml +++ /dev/null @@ -1,21 +0,0 @@ -project: auth-middleware -repo-url: https://github.com/metacall/examples -code-files: - - name: middleware.js - path: examples/auth-middleware/middleware.js - test-cases: - - name: Check the sum function - command: call sum(3,4) - expected-pattern: '7' - - name: Check the signin function - command: call signin("Mostafa","1557") - expected-pattern: '.*' - - name: Check the reverse function - command: call reverse("abcdefg") - expected-pattern: "gfedcba" - - - - - - \ No newline at end of file diff --git a/test-suites/test-string-manipulation.yaml b/test-suites/test-string-manipulation.yaml index f97f070..893e0df 100644 --- a/test-suites/test-string-manipulation.yaml +++ b/test-suites/test-string-manipulation.yaml @@ -5,14 +5,14 @@ code-files: path: examples/string-manipulation/str.rb test-cases: - name: Check the longest_repetition function - command: call longest_repetition("aaa") + function-call: longest_repetition("aaa") expected-pattern: "[ 'a', 3 ]" - name: Check the longest_repetition function with a different string - command: call longest_repetition("bbbaaabaaaa") + function-call: longest_repetition("bbbaaabaaaa") expected-pattern: "[ 'a', 4 ]" - name: Check the longest_repetition function with an empty string - command: call longest_repetition("") - expected-pattern: "NodeJS Loader could not convert the value of type 'Invalid' to N-API | [null, 0]" + function-call: longest_repetition("") + expected-pattern: "NodeJS Loader could not convert the value of type 'Invalid' to N-API || [null, 0]" diff --git a/test-suites/test-time-app-web.yaml b/test-suites/test-time-app-web.yaml index 0a6eb6d..b4a79ed 100644 --- a/test-suites/test-time-app-web.yaml +++ b/test-suites/test-time-app-web.yaml @@ -4,13 +4,13 @@ code-files: - path: examples/time-app-web/index.py test-cases: - name: Check the time is generated in the correct format - command: call time() + function-call: time() expected-pattern: '\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}' - - name: Check calling the time function with a parameter - command: call time(1) - expected-pattern: 'takes 0 positional arguments but 1 was given' + # - name: Check calling the time function with a parameter + # function-call: time(1) + # expected-pattern: 'takes 0 positional arguments but 1 was given' - name: Check index.html is fully returned - command: call index() + function-call: index() expected-pattern: '[\w\W]*' diff --git a/testing.py b/testing.py index 0593782..cf97829 100644 --- a/testing.py +++ b/testing.py @@ -1,13 +1,18 @@ import argparse + +import config + +from testing.logger import Logger from testing.repo_manager import RepoManager -from testing.test_suites_extractor import TestSuitesExtractor from testing.test_runner import TestRunner -from testing.logger import Logger +from testing.test_suites_extractor import TestSuitesExtractor + def main(): parser = argparse.ArgumentParser() parser.add_argument("-f", "--file", action="store", help="the test suite file name") parser.add_argument("-V", "--verbose", action="store_true", help="increase output verbosity") + parser.add_argument("-e", "--environments", nargs="+", default=["cli"], help="the environments to run the tests") args = parser.parse_args() logger = Logger.get_instance() @@ -20,16 +25,18 @@ def main(): if not test_suite_file_name: logger.error("Error: test suite file name is required!") + test_suites_extractor = TestSuitesExtractor(test_suite_file_name) project_name, repo_url, test_suites = test_suites_extractor.extract_test_suites() + config.project_name = project_name logger.info(f"Project: {project_name}") repo_manager = RepoManager(repo_url) repo_manager.clone_repo_if_not_exist() - test_runner = TestRunner(["cli"]) - test_runner.run_tests(project_name, test_suites) + test_runner = TestRunner(args.environments) + test_runner.run_tests(test_suites) if __name__ == "__main__": main() diff --git a/testing/runner/cli_interface.py b/testing/runner/cli_interface.py index f34223c..1087eb3 100644 --- a/testing/runner/cli_interface.py +++ b/testing/runner/cli_interface.py @@ -24,15 +24,16 @@ def get_runtime_tag(self, file_name): else: raise ValueError("Error: file extension not supported!") - def run_test_command(self, file_path, test_case_command): + def run_test_command(self, file_path, function_call): file_name = file_path.split('/')[-1] + function_call = 'call ' + function_call try: if platform.system() == 'Windows': process = subprocess.Popen(['metacall.bat'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) else: process = subprocess.Popen(['metacall'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - commands = ['load ' + ' ' + self.get_runtime_tag(file_name) + ' ' + file_path, test_case_command, 'exit'] + commands = ['load ' + ' ' + self.get_runtime_tag(file_name) + ' ' + file_path, function_call, 'exit'] commands = '\n'.join(commands) + '\n' # join the commands with a newline character process.stdin.write(f"{commands}".encode('utf-8')) diff --git a/testing/runner/faas_interface.py b/testing/runner/faas_interface.py index 4adce55..f430e54 100644 --- a/testing/runner/faas_interface.py +++ b/testing/runner/faas_interface.py @@ -1,13 +1,60 @@ +import json +import subprocess + +import os + from testing.runner.runner_interface import RunnerInterface + class FaaSInterface(RunnerInterface): def __init__(self): - pass + try: + # Get the base URL from the environment variable SERVER_URL + self.base_url = os.environ['SERVER_URL'] + except KeyError: + # If the environment variable is not set, return an error + raise KeyError("SERVER_URL environment variable not set, make sure to run 'source ./deploy.sh /path/to/repo' before running the tests") def get_name(self): return "faas" + + def get_request(self, url): + command = f"curl {url} -X GET" + return command + + def post_request(self, url, params): + try: + params = json.loads(params) + except json.JSONDecodeError: + pass + + data = json.dumps(params) if isinstance(params, dict) else params + command = f"curl {url} -X POST --data '{data}'" + return command + + + def parse_function_call(self, function_call): + if '(' in function_call and ')' in function_call: + function_name = function_call.split('(')[0] + params = function_call.split('(')[1].split(')')[0] + params = params if params else None + else: + function_name = function_call + params = None + + return function_name, params + + def run_test_command(self, file_path, function_call): + function_name, params = self.parse_function_call(function_call) + url = f"{self.base_url}/call/{function_name}" + + if params: + command = self.post_request(url, params) + else: + command = self.get_request(url) + print("Command:", command) + + result = subprocess.run(command, capture_output=True, text=True, shell=True, check=False) + out_str = result.stdout.strip() - def run_test_command(self, file_path, test_case_command): - # Implement the FaaS call here - # For now, return a placeholder string - return "FaaS output placeholder" + return out_str diff --git a/testing/runner/runner_interface.py b/testing/runner/runner_interface.py index be435ca..e7ecfef 100644 --- a/testing/runner/runner_interface.py +++ b/testing/runner/runner_interface.py @@ -2,5 +2,5 @@ class RunnerInterface(ABC): @abstractmethod - def run_test_command(self, file_path, test_case_command): + def run_test_command(self, file_path, function_call): pass diff --git a/testing/test_runner.py b/testing/test_runner.py index 0eb1ee8..d94820b 100644 --- a/testing/test_runner.py +++ b/testing/test_runner.py @@ -5,9 +5,9 @@ class TestCaseGenerator: @staticmethod - def create_test_method(interface, test_case_name, test_case_command, test_case_expected_stdout): + def create_test_method(interface, test_case_name, function_call, test_case_expected_stdout): def test_method(self): - out_str = interface.run_test_command(self.file_path, test_case_command) + out_str = interface.run_test_command(self.file_path, function_call) passed = self.check_match(out_str, test_case_expected_stdout) self.assertTrue(passed, f"{interface.get_name()}_{test_case_name} - Expected: {test_case_expected_stdout}, Actual: {out_str}") return test_method @@ -37,9 +37,9 @@ def check_match(actual, expected_pattern): return TestCaseGenerator.check_match(actual, expected_pattern) for test_case in test_cases: - test_case_name, test_case_command, test_case_expected_stdout = test_case + test_case_name, function_call, test_case_expected_stdout = test_case for interface in self.interfaces: - test_method = TestCaseGenerator.create_test_method(interface, test_case_name, test_case_command, test_case_expected_stdout) + test_method = TestCaseGenerator.create_test_method(interface, test_case_name, function_call, test_case_expected_stdout) test_method_name = f'testCase_{interface.get_name()}_{test_case_name}' setattr(DynamicTestSuite, test_method_name, test_method) @@ -61,7 +61,7 @@ def create_project_test_suites(self, test_suites): master_suite.addTests(test_loader.loadTestsFromTestCase(test_suite)) return master_suite - def run_tests(self, project_name, test_suites): + def run_tests(self, test_suites): master_suite = self.create_project_test_suites(test_suites) runner = unittest.TextTestRunner(verbosity=self.test_verbosity) result = runner.run(master_suite) diff --git a/testing/test_suites_extractor.py b/testing/test_suites_extractor.py index 54e41e2..39dc92b 100644 --- a/testing/test_suites_extractor.py +++ b/testing/test_suites_extractor.py @@ -20,7 +20,7 @@ def extract_test_suites(self): code_files = data['code-files'] test_suites = [] for code_file in code_files: - test_cases = [(test_case['name'], test_case['command'], test_case['expected-pattern']) for test_case in code_file['test-cases']] + test_cases = [(test_case['name'], test_case['function-call'], test_case['expected-pattern']) for test_case in code_file['test-cases']] test_suites.append((code_file['path'], test_cases)) except KeyError as e: self.logger.error(f"Error: parsing yaml file, missing key:{e}")