From 263b46dd7867c2870c1d1a08ed48ab92b935d530 Mon Sep 17 00:00:00 2001 From: Mostafa-wael Date: Mon, 5 Aug 2024 09:44:22 +0300 Subject: [PATCH 1/4] refactor: remove call keyword from the command in the yaml --- README.md | 4 ++-- test-auth-function-mesh.yaml | 19 ------------------- test-auth-middleware.yaml | 21 --------------------- test-suites/test-string-manipulation.yaml | 8 ++++---- test-suites/test-time-app-web.yaml | 6 +++--- testing/runner/cli_interface.py | 5 +++-- testing/runner/faas_interface.py | 2 +- testing/runner/runner_interface.py | 2 +- testing/test_runner.py | 8 ++++---- testing/test_suites_extractor.py | 2 +- 10 files changed, 19 insertions(+), 58 deletions(-) delete mode 100644 test-auth-function-mesh.yaml delete mode 100644 test-auth-middleware.yaml diff --git a/README.md b/README.md index ab5a5b0..d5a639d 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,10 @@ code-files: - path: random-password-generator-example/app.py test-cases: - name: Check the password is generated in the correct length - command: call getRandomPassword(12) + functionCall: getRandomPassword(12) expected-pattern: '\"[\w\W]{12}\"' - name: Check the password is generated in the correct length - command: call getRandomPassword() + functionCall: getRandomPassword() expected-pattern: 'missing 1 required positional argument' ``` diff --git a/test-auth-function-mesh.yaml b/test-auth-function-mesh.yaml deleted file mode 100644 index 593e816..0000000 --- a/test-auth-function-mesh.yaml +++ /dev/null @@ -1,19 +0,0 @@ -project: auth-function-mesh -repo-url: https://github.com/metacall/examples -code-files: - - name: auth.py - path: examples/auth-function-mesh/auth.py - test-cases: - - name: Check the encrypt function - command: call encrypt("asd") - expected-pattern: 'eyJhbGciOiJIUzI1NiJ9.YXNk.QNa-p8QpuHcVUDMN_Ih4x4vidWp31365GM4zrSr3t0s' - - name: Check the decrypt function - command: call decrypt("eyJhbGciOiJIUzI1NiJ9.YXNk.QNa-p8QpuHcVUDMN_Ih4x4vidWp31365GM4zrSr3t0s") - expected-pattern: 'asd' - - - - - - - \ No newline at end of file diff --git a/test-auth-middleware.yaml b/test-auth-middleware.yaml deleted file mode 100644 index e2c8095..0000000 --- a/test-auth-middleware.yaml +++ /dev/null @@ -1,21 +0,0 @@ -project: auth-middleware -repo-url: https://github.com/metacall/examples -code-files: - - name: middleware.js - path: examples/auth-middleware/middleware.js - test-cases: - - name: Check the sum function - command: call sum(3,4) - expected-pattern: '7' - - name: Check the signin function - command: call signin("Mostafa","1557") - expected-pattern: '.*' - - name: Check the reverse function - command: call reverse("abcdefg") - expected-pattern: "gfedcba" - - - - - - \ No newline at end of file diff --git a/test-suites/test-string-manipulation.yaml b/test-suites/test-string-manipulation.yaml index f97f070..4b91fa3 100644 --- a/test-suites/test-string-manipulation.yaml +++ b/test-suites/test-string-manipulation.yaml @@ -5,14 +5,14 @@ code-files: path: examples/string-manipulation/str.rb test-cases: - name: Check the longest_repetition function - command: call longest_repetition("aaa") + functionCall: longest_repetition("aaa") expected-pattern: "[ 'a', 3 ]" - name: Check the longest_repetition function with a different string - command: call longest_repetition("bbbaaabaaaa") + functionCall: longest_repetition("bbbaaabaaaa") expected-pattern: "[ 'a', 4 ]" - name: Check the longest_repetition function with an empty string - command: call longest_repetition("") - expected-pattern: "NodeJS Loader could not convert the value of type 'Invalid' to N-API | [null, 0]" + functionCall: longest_repetition("") + expected-pattern: "NodeJS Loader could not convert the value of type 'Invalid' to N-API || [null, 0]" diff --git a/test-suites/test-time-app-web.yaml b/test-suites/test-time-app-web.yaml index 0a6eb6d..9ce3050 100644 --- a/test-suites/test-time-app-web.yaml +++ b/test-suites/test-time-app-web.yaml @@ -4,13 +4,13 @@ code-files: - path: examples/time-app-web/index.py test-cases: - name: Check the time is generated in the correct format - command: call time() + functionCall: time() expected-pattern: '\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}' - name: Check calling the time function with a parameter - command: call time(1) + functionCall: time(1) expected-pattern: 'takes 0 positional arguments but 1 was given' - name: Check index.html is fully returned - command: call index() + functionCall: index() expected-pattern: '[\w\W]*' diff --git a/testing/runner/cli_interface.py b/testing/runner/cli_interface.py index f34223c..26a60cc 100644 --- a/testing/runner/cli_interface.py +++ b/testing/runner/cli_interface.py @@ -24,15 +24,16 @@ def get_runtime_tag(self, file_name): else: raise ValueError("Error: file extension not supported!") - def run_test_command(self, file_path, test_case_command): + def run_test_command(self, file_path, functionCall): file_name = file_path.split('/')[-1] + functionCall = 'call ' + functionCall try: if platform.system() == 'Windows': process = subprocess.Popen(['metacall.bat'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) else: process = subprocess.Popen(['metacall'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - commands = ['load ' + ' ' + self.get_runtime_tag(file_name) + ' ' + file_path, test_case_command, 'exit'] + commands = ['load ' + ' ' + self.get_runtime_tag(file_name) + ' ' + file_path, functionCall, 'exit'] commands = '\n'.join(commands) + '\n' # join the commands with a newline character process.stdin.write(f"{commands}".encode('utf-8')) diff --git a/testing/runner/faas_interface.py b/testing/runner/faas_interface.py index 4adce55..05e834f 100644 --- a/testing/runner/faas_interface.py +++ b/testing/runner/faas_interface.py @@ -7,7 +7,7 @@ def __init__(self): def get_name(self): return "faas" - def run_test_command(self, file_path, test_case_command): + def run_test_command(self, file_path, functionCall): # Implement the FaaS call here # For now, return a placeholder string return "FaaS output placeholder" diff --git a/testing/runner/runner_interface.py b/testing/runner/runner_interface.py index be435ca..7e1d788 100644 --- a/testing/runner/runner_interface.py +++ b/testing/runner/runner_interface.py @@ -2,5 +2,5 @@ class RunnerInterface(ABC): @abstractmethod - def run_test_command(self, file_path, test_case_command): + def run_test_command(self, file_path, functionCall): pass diff --git a/testing/test_runner.py b/testing/test_runner.py index 0eb1ee8..a5e20a5 100644 --- a/testing/test_runner.py +++ b/testing/test_runner.py @@ -5,9 +5,9 @@ class TestCaseGenerator: @staticmethod - def create_test_method(interface, test_case_name, test_case_command, test_case_expected_stdout): + def create_test_method(interface, test_case_name, functionCall, test_case_expected_stdout): def test_method(self): - out_str = interface.run_test_command(self.file_path, test_case_command) + out_str = interface.run_test_command(self.file_path, functionCall) passed = self.check_match(out_str, test_case_expected_stdout) self.assertTrue(passed, f"{interface.get_name()}_{test_case_name} - Expected: {test_case_expected_stdout}, Actual: {out_str}") return test_method @@ -37,9 +37,9 @@ def check_match(actual, expected_pattern): return TestCaseGenerator.check_match(actual, expected_pattern) for test_case in test_cases: - test_case_name, test_case_command, test_case_expected_stdout = test_case + test_case_name, functionCall, test_case_expected_stdout = test_case for interface in self.interfaces: - test_method = TestCaseGenerator.create_test_method(interface, test_case_name, test_case_command, test_case_expected_stdout) + test_method = TestCaseGenerator.create_test_method(interface, test_case_name, functionCall, test_case_expected_stdout) test_method_name = f'testCase_{interface.get_name()}_{test_case_name}' setattr(DynamicTestSuite, test_method_name, test_method) diff --git a/testing/test_suites_extractor.py b/testing/test_suites_extractor.py index 54e41e2..3865488 100644 --- a/testing/test_suites_extractor.py +++ b/testing/test_suites_extractor.py @@ -20,7 +20,7 @@ def extract_test_suites(self): code_files = data['code-files'] test_suites = [] for code_file in code_files: - test_cases = [(test_case['name'], test_case['command'], test_case['expected-pattern']) for test_case in code_file['test-cases']] + test_cases = [(test_case['name'], test_case['functionCall'], test_case['expected-pattern']) for test_case in code_file['test-cases']] test_suites.append((code_file['path'], test_cases)) except KeyError as e: self.logger.error(f"Error: parsing yaml file, missing key:{e}") From 6da4b97d8c7db283c58b5689982cd60796380bd1 Mon Sep 17 00:00:00 2001 From: Mostafa-wael Date: Thu, 8 Aug 2024 20:40:28 +0300 Subject: [PATCH 2/4] faas working with no ci yet --- README.md | 4 +- test-suites/test-string-manipulation.yaml | 6 +-- test-suites/test-time-app-web.yaml | 10 ++--- testing.py | 15 +++++-- testing/runner/cli_interface.py | 6 +-- testing/runner/faas_interface.py | 52 ++++++++++++++++++++--- testing/runner/runner_interface.py | 2 +- testing/test_runner.py | 10 ++--- testing/test_suites_extractor.py | 2 +- 9 files changed, 78 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index d5a639d..fb480b1 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,10 @@ code-files: - path: random-password-generator-example/app.py test-cases: - name: Check the password is generated in the correct length - functionCall: getRandomPassword(12) + function-call: getRandomPassword(12) expected-pattern: '\"[\w\W]{12}\"' - name: Check the password is generated in the correct length - functionCall: getRandomPassword() + function-call: getRandomPassword() expected-pattern: 'missing 1 required positional argument' ``` diff --git a/test-suites/test-string-manipulation.yaml b/test-suites/test-string-manipulation.yaml index 4b91fa3..893e0df 100644 --- a/test-suites/test-string-manipulation.yaml +++ b/test-suites/test-string-manipulation.yaml @@ -5,13 +5,13 @@ code-files: path: examples/string-manipulation/str.rb test-cases: - name: Check the longest_repetition function - functionCall: longest_repetition("aaa") + function-call: longest_repetition("aaa") expected-pattern: "[ 'a', 3 ]" - name: Check the longest_repetition function with a different string - functionCall: longest_repetition("bbbaaabaaaa") + function-call: longest_repetition("bbbaaabaaaa") expected-pattern: "[ 'a', 4 ]" - name: Check the longest_repetition function with an empty string - functionCall: longest_repetition("") + function-call: longest_repetition("") expected-pattern: "NodeJS Loader could not convert the value of type 'Invalid' to N-API || [null, 0]" diff --git a/test-suites/test-time-app-web.yaml b/test-suites/test-time-app-web.yaml index 9ce3050..b4a79ed 100644 --- a/test-suites/test-time-app-web.yaml +++ b/test-suites/test-time-app-web.yaml @@ -4,13 +4,13 @@ code-files: - path: examples/time-app-web/index.py test-cases: - name: Check the time is generated in the correct format - functionCall: time() + function-call: time() expected-pattern: '\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}' - - name: Check calling the time function with a parameter - functionCall: time(1) - expected-pattern: 'takes 0 positional arguments but 1 was given' + # - name: Check calling the time function with a parameter + # function-call: time(1) + # expected-pattern: 'takes 0 positional arguments but 1 was given' - name: Check index.html is fully returned - functionCall: index() + function-call: index() expected-pattern: '[\w\W]*' diff --git a/testing.py b/testing.py index 0593782..cf97829 100644 --- a/testing.py +++ b/testing.py @@ -1,13 +1,18 @@ import argparse + +import config + +from testing.logger import Logger from testing.repo_manager import RepoManager -from testing.test_suites_extractor import TestSuitesExtractor from testing.test_runner import TestRunner -from testing.logger import Logger +from testing.test_suites_extractor import TestSuitesExtractor + def main(): parser = argparse.ArgumentParser() parser.add_argument("-f", "--file", action="store", help="the test suite file name") parser.add_argument("-V", "--verbose", action="store_true", help="increase output verbosity") + parser.add_argument("-e", "--environments", nargs="+", default=["cli"], help="the environments to run the tests") args = parser.parse_args() logger = Logger.get_instance() @@ -20,16 +25,18 @@ def main(): if not test_suite_file_name: logger.error("Error: test suite file name is required!") + test_suites_extractor = TestSuitesExtractor(test_suite_file_name) project_name, repo_url, test_suites = test_suites_extractor.extract_test_suites() + config.project_name = project_name logger.info(f"Project: {project_name}") repo_manager = RepoManager(repo_url) repo_manager.clone_repo_if_not_exist() - test_runner = TestRunner(["cli"]) - test_runner.run_tests(project_name, test_suites) + test_runner = TestRunner(args.environments) + test_runner.run_tests(test_suites) if __name__ == "__main__": main() diff --git a/testing/runner/cli_interface.py b/testing/runner/cli_interface.py index 26a60cc..1087eb3 100644 --- a/testing/runner/cli_interface.py +++ b/testing/runner/cli_interface.py @@ -24,16 +24,16 @@ def get_runtime_tag(self, file_name): else: raise ValueError("Error: file extension not supported!") - def run_test_command(self, file_path, functionCall): + def run_test_command(self, file_path, function_call): file_name = file_path.split('/')[-1] - functionCall = 'call ' + functionCall + function_call = 'call ' + function_call try: if platform.system() == 'Windows': process = subprocess.Popen(['metacall.bat'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) else: process = subprocess.Popen(['metacall'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - commands = ['load ' + ' ' + self.get_runtime_tag(file_name) + ' ' + file_path, functionCall, 'exit'] + commands = ['load ' + ' ' + self.get_runtime_tag(file_name) + ' ' + file_path, function_call, 'exit'] commands = '\n'.join(commands) + '\n' # join the commands with a newline character process.stdin.write(f"{commands}".encode('utf-8')) diff --git a/testing/runner/faas_interface.py b/testing/runner/faas_interface.py index 05e834f..eed3797 100644 --- a/testing/runner/faas_interface.py +++ b/testing/runner/faas_interface.py @@ -1,13 +1,55 @@ +import json +import subprocess + +import config + from testing.runner.runner_interface import RunnerInterface + class FaaSInterface(RunnerInterface): def __init__(self): - pass + self.base_url = "http://localhost:9000/9b150a863eeb" def get_name(self): return "faas" + + def get_request(self, url): + command = f"curl {url} -X GET" + return command + + def post_request(self, url, params): + try: + params = json.loads(params) + except json.JSONDecodeError: + pass + + data = json.dumps(params) if isinstance(params, dict) else params + command = f"curl {url} -X POST --data '{data}'" + return command + + + def parse_function_call(self, function_call): + if '(' in function_call and ')' in function_call: + function_name = function_call.split('(')[0] + params = function_call.split('(')[1].split(')')[0] + params = params if params else None + else: + function_name = function_call + params = None + + return function_name, params + + def run_test_command(self, file_path, function_call): + function_name, params = self.parse_function_call(function_call) + url = f"{self.base_url}/{config.project_name}/v1/call/{function_name}" + + if params: + command = self.post_request(url, params) + else: + command = self.get_request(url) + print("Command:", command) + + result = subprocess.run(command, capture_output=True, text=True, shell=True, check=False) + out_str = result.stdout.strip() - def run_test_command(self, file_path, functionCall): - # Implement the FaaS call here - # For now, return a placeholder string - return "FaaS output placeholder" + return out_str diff --git a/testing/runner/runner_interface.py b/testing/runner/runner_interface.py index 7e1d788..e7ecfef 100644 --- a/testing/runner/runner_interface.py +++ b/testing/runner/runner_interface.py @@ -2,5 +2,5 @@ class RunnerInterface(ABC): @abstractmethod - def run_test_command(self, file_path, functionCall): + def run_test_command(self, file_path, function_call): pass diff --git a/testing/test_runner.py b/testing/test_runner.py index a5e20a5..d94820b 100644 --- a/testing/test_runner.py +++ b/testing/test_runner.py @@ -5,9 +5,9 @@ class TestCaseGenerator: @staticmethod - def create_test_method(interface, test_case_name, functionCall, test_case_expected_stdout): + def create_test_method(interface, test_case_name, function_call, test_case_expected_stdout): def test_method(self): - out_str = interface.run_test_command(self.file_path, functionCall) + out_str = interface.run_test_command(self.file_path, function_call) passed = self.check_match(out_str, test_case_expected_stdout) self.assertTrue(passed, f"{interface.get_name()}_{test_case_name} - Expected: {test_case_expected_stdout}, Actual: {out_str}") return test_method @@ -37,9 +37,9 @@ def check_match(actual, expected_pattern): return TestCaseGenerator.check_match(actual, expected_pattern) for test_case in test_cases: - test_case_name, functionCall, test_case_expected_stdout = test_case + test_case_name, function_call, test_case_expected_stdout = test_case for interface in self.interfaces: - test_method = TestCaseGenerator.create_test_method(interface, test_case_name, functionCall, test_case_expected_stdout) + test_method = TestCaseGenerator.create_test_method(interface, test_case_name, function_call, test_case_expected_stdout) test_method_name = f'testCase_{interface.get_name()}_{test_case_name}' setattr(DynamicTestSuite, test_method_name, test_method) @@ -61,7 +61,7 @@ def create_project_test_suites(self, test_suites): master_suite.addTests(test_loader.loadTestsFromTestCase(test_suite)) return master_suite - def run_tests(self, project_name, test_suites): + def run_tests(self, test_suites): master_suite = self.create_project_test_suites(test_suites) runner = unittest.TextTestRunner(verbosity=self.test_verbosity) result = runner.run(master_suite) diff --git a/testing/test_suites_extractor.py b/testing/test_suites_extractor.py index 3865488..39dc92b 100644 --- a/testing/test_suites_extractor.py +++ b/testing/test_suites_extractor.py @@ -20,7 +20,7 @@ def extract_test_suites(self): code_files = data['code-files'] test_suites = [] for code_file in code_files: - test_cases = [(test_case['name'], test_case['functionCall'], test_case['expected-pattern']) for test_case in code_file['test-cases']] + test_cases = [(test_case['name'], test_case['function-call'], test_case['expected-pattern']) for test_case in code_file['test-cases']] test_suites.append((code_file['path'], test_cases)) except KeyError as e: self.logger.error(f"Error: parsing yaml file, missing key:{e}") From 5a0cdf7a19e83634e4b944c0a59bc586cc9c4dc8 Mon Sep 17 00:00:00 2001 From: Mostafa-wael Date: Fri, 9 Aug 2024 18:01:03 +0300 Subject: [PATCH 3/4] Fix the pipeline and create the deploy script --- .github/workflows/test.yml | 12 ++--- deploy.sh | 76 ++++++++++++++++++++++++++++++++ requirements.txt | 3 ++ testing/runner/faas_interface.py | 11 +++-- 4 files changed, 93 insertions(+), 9 deletions(-) create mode 100644 deploy.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 29b1afc..fe78598 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,8 +7,8 @@ on: branches: - main -jobs: - Linux: +jobs: + Linux_cli: name: Linux - Ubuntu Run runs-on: ubuntu-latest steps: @@ -36,9 +36,9 @@ jobs: - name: Run Tests Suits run: | - find test-suites -type f -name "*.yaml" -exec python ./testing.py -f {} -V \; + find test-suites -type f -name "*.yaml" -exec python ./testing.py -f {} -V -e cli \; - Windows: + Windows_cli: name: Windows Run runs-on: windows-latest steps: @@ -74,7 +74,7 @@ jobs: - name: Run Test Suits shell: bash run: | - find test-suites -type f -name "*.yaml" -exec python ./testing.py -f {} -V \; + find test-suites -type f -name "*.yaml" -exec python ./testing.py -f {} -V -e cli \; # MacOS: @@ -100,4 +100,4 @@ jobs: # - name: Run Tests Suits # run: | - # find test-suites -type f -name "*.yaml" -exec python ./testing.py -f {} -V \; + # find test-suites -type f -name "*.yaml" -exec python ./testing.py -f {} -V -e cli \; diff --git a/deploy.sh b/deploy.sh new file mode 100644 index 0000000..000ebe3 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +# Accept the repo path as an argument +repo_path=$1 + +# Run the metacall-deploy command and store the output +output=$(metacall-deploy --inspect OpenAPIv3 --dev --workdir "$repo_path") + +# Parse the JSON output using jq to extract the server URL and paths +server_url=$(echo "$output" | jq -r '.[0].servers[0].url') +paths=$(echo "$output" | jq -r '.[0].paths | keys[]') + +# Output the server URL and paths +echo "Server URL: $server_url" +echo "Available Paths:" +for path in $paths; do + echo "$server_url$path" +done + +# Set the server URL as an environment variable +export SERVER_URL=$server_url + + +# multi line comment +: ' Exmaple output +[ + { + "openapi": "3.0.0", + "info": { + "title": "MetaCall Cloud FaaS deployment 'time-app-web'", + "description": "", + "version": "v1" + }, + "servers": [ + { + "url": "http://localhost:9000/aa759149a70a/time-app-web/v1", + "description": "MetaCall Cloud FaaS" + } + ], + "paths": { + "/call/time": { + "get": { + "summary": "", + "description": "", + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + }, + "/call/index": { + "get": { + "summary": "", + "description": "", + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + } + } + } +] +' \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index be2b74d..767fd22 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,4 @@ +config==0.5.1 +metacall==0.5.0 PyYAML==6.0.1 +PyYAML==6.0.2 diff --git a/testing/runner/faas_interface.py b/testing/runner/faas_interface.py index eed3797..f430e54 100644 --- a/testing/runner/faas_interface.py +++ b/testing/runner/faas_interface.py @@ -1,14 +1,19 @@ import json import subprocess -import config +import os from testing.runner.runner_interface import RunnerInterface class FaaSInterface(RunnerInterface): def __init__(self): - self.base_url = "http://localhost:9000/9b150a863eeb" + try: + # Get the base URL from the environment variable SERVER_URL + self.base_url = os.environ['SERVER_URL'] + except KeyError: + # If the environment variable is not set, return an error + raise KeyError("SERVER_URL environment variable not set, make sure to run 'source ./deploy.sh /path/to/repo' before running the tests") def get_name(self): return "faas" @@ -41,7 +46,7 @@ def parse_function_call(self, function_call): def run_test_command(self, file_path, function_call): function_name, params = self.parse_function_call(function_call) - url = f"{self.base_url}/{config.project_name}/v1/call/{function_name}" + url = f"{self.base_url}/call/{function_name}" if params: command = self.post_request(url, params) From d6fb76d0651c1602dc8c720897a4d1986a772821 Mon Sep 17 00:00:00 2001 From: Vicente Eduardo Ferrer Garcia <7854099+viferga@users.noreply.github.com> Date: Fri, 9 Aug 2024 11:03:00 -0400 Subject: [PATCH 4/4] Update requirements.txt --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 767fd22..217629d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ config==0.5.1 metacall==0.5.0 -PyYAML==6.0.1 PyYAML==6.0.2