From 52f86a92bcb43ba138cdd420ae841ff0fa3a711e Mon Sep 17 00:00:00 2001 From: bidaya0 Date: Fri, 16 Jun 2023 16:46:38 +0800 Subject: [PATCH 001/161] feat: add project id and project version id in project add api. --- dongtai_protocol/api_schema.py | 52 +++++++++++++++++++++++- dongtai_protocol/views/agent_download.py | 40 +++++++++++------- dongtai_web/views/project_add.py | 39 ++++++++++++------ 3 files changed, 101 insertions(+), 30 deletions(-) diff --git a/dongtai_protocol/api_schema.py b/dongtai_protocol/api_schema.py index 19f00b887..2eb3ecb0e 100644 --- a/dongtai_protocol/api_schema.py +++ b/dongtai_protocol/api_schema.py @@ -27,6 +27,7 @@ class DongTaiParameter: PROJECT_NAME = OpenApiParameter( name='projectName', type=str, + required=True, description='The name of the project where the Agent needs to be installed', examples=[ OpenApiExample( @@ -37,10 +38,56 @@ class DongTaiParameter: ], ) + PROJECT_VERSION = OpenApiParameter( + name='projectVersion', + type=str, + required=True, + description= + 'The version name of the project where the Agent needs to be installed', + examples=[ + OpenApiExample( + 'example with https://iast.io', + summary='default', + value='Demo Project', + ), + ], + ) + + DEPARTMENT_TOKEN = OpenApiParameter( + name='department_token', + type=str, + required=True, + description= + 'The department token where the Agent needs to be installed', + examples=[ + OpenApiExample( + 'example with https://iast.io', + summary='default', + value='Demo Project', + ), + ], + ) + + TEMPLATE_ID = OpenApiParameter( + name='template_id', + type=int, + required=True, + description= + 'The project template_id where the Agent needs to be installed, set 1 as default', + examples=[ + OpenApiExample( + 'the default value', + summary='default', + value=1, + ), + ], + ) + LANGUAGE = OpenApiParameter( name='language', type=str, - description='The development language of the project that needs to install the Agent', + description= + 'The development language of the project that needs to install the Agent', required=True, examples=[ OpenApiExample( @@ -54,7 +101,8 @@ class DongTaiParameter: VERSION = OpenApiParameter( name='version', type=str, - description='The development language of the project that needs to install the Agent', + description= + 'The development language of the project that needs to install the Agent', required=True, examples=[ OpenApiExample( diff --git a/dongtai_protocol/views/agent_download.py b/dongtai_protocol/views/agent_download.py index ebe657490..bdc58d485 100644 --- a/dongtai_protocol/views/agent_download.py +++ b/dongtai_protocol/views/agent_download.py @@ -272,8 +272,9 @@ def replace_config(self): class AgentDownload(OpenApiEndPoint): """ - 当前用户详情 + Agent 下载接口 """ + name = "download_iast_agent" description = "下载洞态Agent" authentication_classes = (DepartmentTokenAuthentication, @@ -306,21 +307,27 @@ def make_download_handler(self, language, user_id): return GoAgentDownload(user_id) return - @extend_schema( - parameters=[ - DongTaiParameter.OPENAPI_URL, - DongTaiParameter.PROJECT_NAME, - DongTaiParameter.LANGUAGE - ], - auth=[DongTaiAuth.TOKEN], - responses=[FileResponse], - methods=['GET'] - ) + @extend_schema(operation_id="agent download api", + tags=[_('Agent Protocol')], + summary=_('Agent download'), + parameters=[ + DongTaiParameter.OPENAPI_URL, + DongTaiParameter.PROJECT_NAME, + DongTaiParameter.PROJECT_VERSION, + DongTaiParameter.TEMPLATE_ID, + DongTaiParameter.DEPARTMENT_TOKEN, + DongTaiParameter.LANGUAGE, + ], + responses=[FileResponse], + methods=['GET']) def get(self, request): try: - base_url = request.query_params.get('url', 'https://www.huoxian.cn') - project_name = request.query_params.get('projectName', 'Demo Project') - project_version = request.query_params.get('projectVersion', 'V1.0') + base_url = request.query_params.get('url', + 'https://www.huoxian.cn') + project_name = request.query_params.get('projectName', + 'Demo Project') + project_version = request.query_params.get('projectVersion', + 'V1.0') language = request.query_params.get('language') department_token = request.query_params.get('department_token') template_id = request.query_params.get('template_id') @@ -337,7 +344,10 @@ def get(self, request): handler = self.make_download_handler(language, request.user.id) if handler.download_agent() is False: - return R.failure(msg="agent file download failure. please contact official staff for help.") + return R.failure( + msg= + "agent file download failure. please contact official staff for help." + ) if handler.create_config( base_url=base_url, diff --git a/dongtai_web/views/project_add.py b/dongtai_web/views/project_add.py index 2c809f911..f2c8cb9d5 100644 --- a/dongtai_web/views/project_add.py +++ b/dongtai_web/views/project_add.py @@ -31,19 +31,23 @@ class _ProjectsAddBodyArgsSerializer(serializers.Serializer): name = serializers.CharField(help_text=_('The name of project')) - scan_id = serializers.IntegerField( - help_text=_("The id corresponding to the scanning strategy.")) + template_id = serializers.IntegerField(help_text=_( + "The id corresponding to the project template. required to specfic, use 1 as default." + )) version_name = serializers.CharField( - help_text=_("The version name of the project")) - pid = serializers.IntegerField(help_text=_("The id of the project")) + required=False, help_text=_("The version name of the project")) + pid = serializers.IntegerField( + required=False, + help_text=_( + "The id of the project, use it when try to modify existed project." + )) description = serializers.CharField( - help_text=_("Description of the project")) + required=False, help_text=_("Description of the project")) vul_validation = serializers.IntegerField( - help_text="vul validation switch") - base_url = serializers.CharField() - test_req_header_key = serializers.CharField() - test_req_header_value = serializers.CharField() - template_id = serializers.IntegerField(help_text=_("The id of the project")) + help_text="vul validation switch", ) + base_url = serializers.CharField(required=False, ) + test_req_header_key = serializers.CharField(required=False, ) + test_req_header_value = serializers.CharField(required=False, ) _ResponseSerializer = get_response_serializer(status_msg_keypair=( @@ -128,8 +132,11 @@ def post(self, request): project = IastProject.objects.filter( name=name, user_id=request.user.id, department_id=department_id).first() if not project: - project = IastProject.objects.create(name=name, - user_id=request.user.id, department_id=department_id, template_id=template_id) + project = IastProject.objects.create( + name=name, + user_id=request.user.id, + department_id=department_id, + template_id=template_id) else: return R.failure( status=203, @@ -187,7 +194,13 @@ def post(self, request): 'template_id', 'department_id', 'enable_log', 'log_level' ]) disable_cache(get_scan_id, (project.id)) - return R.success(msg='操作成功') + return R.success( + data={ + "project_id": project.id, + "project_version_id": project_version_id + }, + msg='操作成功', + ) except Exception as e: logger.error(e, exc_info=e) return R.failure(status=202, msg=_('Parameter error')) From b4fbe80dea1b2131efd5235fc5be2c67c6540115 Mon Sep 17 00:00:00 2001 From: bidaya0 Date: Fri, 16 Jun 2023 17:22:30 +0800 Subject: [PATCH 002/161] feat: add project id and project version id in project add api. --- dongtai_protocol/api_schema.py | 15 +++++---------- dongtai_protocol/views/agent_download.py | 3 +-- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/dongtai_protocol/api_schema.py b/dongtai_protocol/api_schema.py index 2eb3ecb0e..5c1619878 100644 --- a/dongtai_protocol/api_schema.py +++ b/dongtai_protocol/api_schema.py @@ -42,8 +42,7 @@ class DongTaiParameter: name='projectVersion', type=str, required=True, - description= - 'The version name of the project where the Agent needs to be installed', + description='The version name of the project where the Agent needs to be installed', examples=[ OpenApiExample( 'example with https://iast.io', @@ -57,8 +56,7 @@ class DongTaiParameter: name='department_token', type=str, required=True, - description= - 'The department token where the Agent needs to be installed', + description='The department token where the Agent needs to be installed', examples=[ OpenApiExample( 'example with https://iast.io', @@ -72,8 +70,7 @@ class DongTaiParameter: name='template_id', type=int, required=True, - description= - 'The project template_id where the Agent needs to be installed, set 1 as default', + description='The project template_id where the Agent needs to be installed, set 1 as default', examples=[ OpenApiExample( 'the default value', @@ -86,8 +83,7 @@ class DongTaiParameter: LANGUAGE = OpenApiParameter( name='language', type=str, - description= - 'The development language of the project that needs to install the Agent', + description='The development language of the project that needs to install the Agent', required=True, examples=[ OpenApiExample( @@ -101,8 +97,7 @@ class DongTaiParameter: VERSION = OpenApiParameter( name='version', type=str, - description= - 'The development language of the project that needs to install the Agent', + description='The development language of the project that needs to install the Agent', required=True, examples=[ OpenApiExample( diff --git a/dongtai_protocol/views/agent_download.py b/dongtai_protocol/views/agent_download.py index bdc58d485..9bc32505c 100644 --- a/dongtai_protocol/views/agent_download.py +++ b/dongtai_protocol/views/agent_download.py @@ -345,8 +345,7 @@ def get(self, request): if handler.download_agent() is False: return R.failure( - msg= - "agent file download failure. please contact official staff for help." + msg="agent file download failure. please contact official staff for help." ) if handler.create_config( From febaed3553d871764bc9a7295f1ac3ca398cbd77 Mon Sep 17 00:00:00 2001 From: bidaya0 Date: Mon, 19 Jun 2023 16:17:19 +0800 Subject: [PATCH 003/161] fix: api route parameter not correct record with multi method case. --- .github/workflows/teststate.yml | 2 + dongtai_conf/celery.py | 1 + .../report/handler/api_route_handler.py | 1 + .../test_agent_apiroute_parameter.py | 51 +++++++++++++++++++ 4 files changed, 55 insertions(+) create mode 100644 test/apiserver/test_agent_apiroute_parameter.py diff --git a/.github/workflows/teststate.yml b/.github/workflows/teststate.yml index 4752ce04b..79e204b48 100644 --- a/.github/workflows/teststate.yml +++ b/.github/workflows/teststate.yml @@ -244,6 +244,7 @@ jobs: - name: Django Unit Testing run: | + export CELERY_EAGER_TEST=TRUE cp dongtai_conf/conf/config.ini.test dongtai_conf/conf/config.ini mkdir -p /tmp/logstash/report/{img,word,pdf,excel} && mkdir -p /tmp/iast_cache/package && mkdir -p /tmp/logstash/batchagent python3 ./deploy/docker/version_update.py || true @@ -293,6 +294,7 @@ jobs: - name: Django Unit Testing run: | + export CELERY_EAGER_TEST=TRUE cp dongtai_conf/conf/config.ini.test dongtai_conf/conf/config.ini mkdir -p /tmp/logstash/report/{img,word,pdf,excel} && mkdir -p /tmp/iast_cache/package && mkdir -p /tmp/logstash/batchagent python3 ./deploy/docker/version_update.py || true diff --git a/dongtai_conf/celery.py b/dongtai_conf/celery.py index 035ac035e..e3c776253 100644 --- a/dongtai_conf/celery.py +++ b/dongtai_conf/celery.py @@ -58,6 +58,7 @@ configs['accept_content'] = ['json'] configs['task_ignore_result'] = True configs['task_acks_late'] = True +configs['task_always_eager'] = True if os.getenv("CELERY_EAGER_TEST") == "TRUE" else False configs['task_acks_on_failure_or_timeout'] = True configs['broker_connection_retry_on_startup'] = False configs['broker_connection_max_retries'] = 0 # it means retry forever diff --git a/dongtai_protocol/report/handler/api_route_handler.py b/dongtai_protocol/report/handler/api_route_handler.py index 36f1fb00a..40a91c025 100644 --- a/dongtai_protocol/report/handler/api_route_handler.py +++ b/dongtai_protocol/report/handler/api_route_handler.py @@ -103,6 +103,7 @@ def _route_dump(item, api_method, agent): def _para_dump(item, api_route): + item = item.copy() item['route'] = api_route item['parameter_type'] = item['type'] del item['type'] diff --git a/test/apiserver/test_agent_apiroute_parameter.py b/test/apiserver/test_agent_apiroute_parameter.py new file mode 100644 index 000000000..d40d367cd --- /dev/null +++ b/test/apiserver/test_agent_apiroute_parameter.py @@ -0,0 +1,51 @@ +from test.apiserver.test_agent_base import AgentTestCase +from dongtai_common.models.api_route import IastApiParameter, IastApiRoute + +data = { + "type": 97, + "detail": { + "agentId": + 8, + "apiData": [{ + "controller": + "app.iast.api.springmvc.controller.GreetingController", + "file": + "", + "method": ["GET", "POST"], + "description": + "", + "uri": + "/request-mapping/path/{value1}/{value2}", + "class": + "app.iast.api.springmvc.controller.GreetingController", + "parameters": [{ + "annotation": "restful访问参数", + "name": "value1", + "type": "java.lang.String" + }, { + "annotation": "restful访问参数", + "name": "value2", + "type": "java.lang.String" + }, { + "annotation": "", + "name": "model", + "type": "org.springframework.ui.Model" + }], + "returnType": + "java.lang.String" + }] + } +} + + +class ApiRouteParameterCheckTestCase(AgentTestCase): + + def test_agent_api_upload(self): + data['detail']['agentId'] = self.agent_id + res = self.agent_report(data, agentId=self.agent_id) + api_routes = list( + IastApiRoute.objects.filter( + path="/request-mapping/path/{value1}/{value2}").all()) + self.assertEqual(len(api_routes), 2) + for route in api_routes: + self.assertEqual(route.iastapiparameter_set.count(), 3) From 737979b6de2eb298ba59a91c36d254291b6d341f Mon Sep 17 00:00:00 2001 From: bidaya0 Date: Mon, 19 Jun 2023 16:21:47 +0800 Subject: [PATCH 004/161] fix: api route parameter not correct record with multi method case. --- dongtai_conf/celery.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dongtai_conf/celery.py b/dongtai_conf/celery.py index e3c776253..1c82272b2 100644 --- a/dongtai_conf/celery.py +++ b/dongtai_conf/celery.py @@ -58,7 +58,8 @@ configs['accept_content'] = ['json'] configs['task_ignore_result'] = True configs['task_acks_late'] = True -configs['task_always_eager'] = True if os.getenv("CELERY_EAGER_TEST") == "TRUE" else False +configs['task_always_eager'] = True if os.getenv( + "CELERY_EAGER_TEST") == "TRUE" else False configs['task_acks_on_failure_or_timeout'] = True configs['broker_connection_retry_on_startup'] = False configs['broker_connection_max_retries'] = 0 # it means retry forever From f3301c18531e1e9d5b7e9204b24e8ac6fa9bf8be Mon Sep 17 00:00:00 2001 From: bidaya0 Date: Mon, 19 Jun 2023 16:36:39 +0800 Subject: [PATCH 005/161] fix: api route parameter not correct record with multi method case. --- dongtai_protocol/tests.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/dongtai_protocol/tests.py b/dongtai_protocol/tests.py index 5be22cae5..ada6dc4c5 100644 --- a/dongtai_protocol/tests.py +++ b/dongtai_protocol/tests.py @@ -19,6 +19,7 @@ from django.test import TestCase +@unittest.skip("waiting for rebuild mock data") class AgentMethodPoolUploadTestCase(AgentTestCase): def test_benchmark_agent_method_pool_upload(self): @@ -36,7 +37,8 @@ def test_benchmark_agent_method_pool_upload(self): assert res == "" assert MethodPool.objects.filter( url=report['detail']["url"]).exists() - assert MethodPool.objects.filter(agent_id=self.agent_id).count() == len(data) + assert MethodPool.objects.filter( + agent_id=self.agent_id).count() == len(data) def download_file(url, filepath): @@ -70,10 +72,11 @@ def test_agent_replay_queryset(self): self.agent.server.hostname, language=self.agent.language)) replay_queryset = IastReplayQueue.objects.values( - 'id', 'relation_id', 'uri', 'method', 'scheme', 'header', - 'params', 'body', 'replay_type').filter( - agent_id__in=project_agents, - state__in=[const.WAITING, const.SOLVING])[:200] + 'id', 'relation_id', 'uri', 'method', 'scheme', 'header', 'params', + 'body', + 'replay_type').filter(agent_id__in=project_agents, + state__in=[const.WAITING, + const.SOLVING])[:200] def test_agent_replay_queryset_result(self): self.agent = IastAgent.objects.filter(pk=self.agent_id).first() @@ -94,6 +97,7 @@ def get_replay_id_set(replay_list: list) -> set: @unittest.skip("waiting for rebuild mock data") class AgentSaasMethodPoolParseApiTestCase(AgentTestCase): + def test_api_parse(self): mp = MethodPool.objects.filter(pk=500483715).first() mp.req_header From 95eda3d71c07f3dc79a2bfa3d139c9ff7479112e Mon Sep 17 00:00:00 2001 From: bidaya0 Date: Mon, 19 Jun 2023 16:41:48 +0800 Subject: [PATCH 006/161] fix: api route parameter not correct record with multi method case. --- dongtai_protocol/report/handler/api_route_handler.py | 1 + test/apiserver/test_agent_apiroute_parameter.py | 1 + 2 files changed, 2 insertions(+) diff --git a/dongtai_protocol/report/handler/api_route_handler.py b/dongtai_protocol/report/handler/api_route_handler.py index 40a91c025..20859ccab 100644 --- a/dongtai_protocol/report/handler/api_route_handler.py +++ b/dongtai_protocol/report/handler/api_route_handler.py @@ -111,6 +111,7 @@ def _para_dump(item, api_route): def _response_dump(item, api_route): + item = item.copy() item['route'] = api_route return item diff --git a/test/apiserver/test_agent_apiroute_parameter.py b/test/apiserver/test_agent_apiroute_parameter.py index d40d367cd..aa4504c44 100644 --- a/test/apiserver/test_agent_apiroute_parameter.py +++ b/test/apiserver/test_agent_apiroute_parameter.py @@ -49,3 +49,4 @@ def test_agent_api_upload(self): self.assertEqual(len(api_routes), 2) for route in api_routes: self.assertEqual(route.iastapiparameter_set.count(), 3) + self.assertEqual(route.iastapiresponse_set.count(), 1) From d142b2d253c6d0d612452f344ba7628d71ca7348 Mon Sep 17 00:00:00 2001 From: bidaya0 Date: Wed, 21 Jun 2023 16:24:13 +0800 Subject: [PATCH 007/161] feat/new_api_route_upload --- dongtai_common/models/api_route_v2.py | 40 +++++++ dongtai_common/utils/const.py | 1 + dongtai_conf/celery.py | 1 + dongtai_protocol/report/__init__.py | 5 +- .../report/handler/api_route_v2_handler.py | 104 ++++++++++++++++++ .../handler/saas_method_pool_handler.py | 2 +- test/apiserver/test_agent_apiroute_v2.py | 16 +++ test/integration/mockdata/api-report.json | 1 + 8 files changed, 168 insertions(+), 2 deletions(-) create mode 100644 dongtai_common/models/api_route_v2.py create mode 100644 dongtai_protocol/report/handler/api_route_v2_handler.py create mode 100644 test/apiserver/test_agent_apiroute_v2.py create mode 100644 test/integration/mockdata/api-report.json diff --git a/dongtai_common/models/api_route_v2.py b/dongtai_common/models/api_route_v2.py new file mode 100644 index 000000000..55a9931db --- /dev/null +++ b/dongtai_common/models/api_route_v2.py @@ -0,0 +1,40 @@ +###################################################################### +# @author : bidaya0 (bidaya0@$HOSTNAME) +# @file : api_route +# @created : Tuesday Aug 17, 2021 17:43:27 CST +# +# @description : +###################################################################### + +from django.db import models +from dongtai_common.utils.settings import get_managed +from dongtai_common.models.agent import IastAgent +from dongtai_common.models.project import IastProject +from dongtai_common.models.project_version import IastProjectVersion + + +class FromWhereChoices(models.IntegerChoices): + FROM_AGENT = 1 + + +class IastApiRouteV2(models.Model): + path = models.CharField(max_length=255, blank=True) + method = models.CharField(max_length=100, blank=True) + from_where = models.IntegerField(default=FromWhereChoices.FROM_AGENT, + choices=FromWhereChoices.choices) + project = models.ForeignKey(IastProject, + on_delete=models.CASCADE, + blank=True, + null=True, + default=-1) + project_version = models.ForeignKey(IastProjectVersion, + on_delete=models.CASCADE, + blank=True, + null=True, + default=-1) + is_cover = models.IntegerField(default=0) + api_info = models.JSONField(blank=True, null=True, default=dict) + + class Meta: + managed = get_managed() + db_table = 'iast_api_route_v2' diff --git a/dongtai_common/utils/const.py b/dongtai_common/utils/const.py index d4ac70f27..5eb6d65aa 100644 --- a/dongtai_common/utils/const.py +++ b/dongtai_common/utils/const.py @@ -16,6 +16,7 @@ REPORT_AUTH_UPDATE = 0x32 REPORT_ERROR_LOG = 0x51 REPORT_API_ROUTE = 0x61 +REPORT_API_ROUTE_V2 = 0x62 REPORT_THIRD_PARTY_SERVICE = 0x81 REPORT_FILE_PATH = 0x82 diff --git a/dongtai_conf/celery.py b/dongtai_conf/celery.py index 1c82272b2..4be740b19 100644 --- a/dongtai_conf/celery.py +++ b/dongtai_conf/celery.py @@ -68,6 +68,7 @@ # configs['worker_concurrency'] = 8 configs["task_routes"] = { # normal + "dongtai_protocol.report.handler.api_route_v2_handler.api_route_gather_v2": {'queue': 'dongtai-api-route-handler', 'routing_key': 'dongtai-api-route-handler'}, "dongtai_protocol.report.handler.api_route_handler.api_route_gather": {'queue': 'dongtai-api-route-handler', 'routing_key': 'dongtai-api-route-handler'}, "dongtai_engine.tasks.search_vul_from_method_pool": {'queue': 'dongtai-method-pool-scan', 'routing_key': 'dongtai-method-pool-scan'}, "dongtai_engine.plugins.project_time_update.project_time_stamp_update": {'queue': 'dongtai-project-time-stamp-update', 'routing_key': 'dongtai-project-time-stamp-update'}, diff --git a/dongtai_protocol/report/__init__.py b/dongtai_protocol/report/__init__.py index 18cd8cf2e..1aac0b5fa 100644 --- a/dongtai_protocol/report/__init__.py +++ b/dongtai_protocol/report/__init__.py @@ -8,8 +8,10 @@ from dongtai_protocol.report.handler.heartbeat_handler import HeartBeatHandler from dongtai_protocol.report.handler.narmal_vul_handler import NormalVulnHandler from dongtai_protocol.report.handler.saas_method_pool_handler import SaasMethodPoolHandler -from dongtai_protocol.report.handler.sca_handler import (ScaHandler, ScaBulkHandler) +from dongtai_protocol.report.handler.sca_handler import (ScaHandler, + ScaBulkHandler) from dongtai_protocol.report.handler.api_route_handler import ApiRouteHandler +from dongtai_protocol.report.handler.api_route_v2_handler import ApiRouteV2Handler from dongtai_protocol.report.handler.hardencode_vul_handler import HardEncodeVulHandler from dongtai_protocol.report.handler.agent_third_service_handler import ThirdPartyServiceHandler from dongtai_protocol.report.handler.agent_filepath_handler import FilePathHandler @@ -21,6 +23,7 @@ NormalVulnHandler() SaasMethodPoolHandler() ApiRouteHandler() + ApiRouteV2Handler() HardEncodeVulHandler() ScaBulkHandler() ThirdPartyServiceHandler() diff --git a/dongtai_protocol/report/handler/api_route_v2_handler.py b/dongtai_protocol/report/handler/api_route_v2_handler.py new file mode 100644 index 000000000..27a4a5675 --- /dev/null +++ b/dongtai_protocol/report/handler/api_route_v2_handler.py @@ -0,0 +1,104 @@ +###################################################################### +# @author : bidaya0 (bidaya0@$HOSTNAME) +# @file : api_route_handler +# @created : Tuesday Aug 17, 2021 19:59:29 CST +# +# @description : +###################################################################### + +from dongtai_protocol.report.handler.report_handler_interface import IReportHandler +from dongtai_protocol.report.report_handler_factory import ReportHandler +from dongtai_common.models.api_route_v2 import IastApiRouteV2, FromWhereChoices +from dongtai_common.models.agent import IastAgent +from dongtai_common.utils import const +import logging +from django.utils.translation import gettext_lazy as _ +from django.db import transaction +from dongtai_common.models.project import IastProject +from dongtai_engine.plugins.project_time_update import project_time_stamp_update +from celery import shared_task +import re + +logger = logging.getLogger('dongtai.openapi') + + +@ReportHandler.register(const.REPORT_API_ROUTE_V2) +class ApiRouteV2Handler(IReportHandler): + + def parse(self): + self.api_data = self.detail.get('apiData') + + def save(self): + api_route_gather_v2.delay(self.agent_id, self.api_data) + + +def find_values_with_key(dictionary, key): + values = [] + for dic_key, value in dictionary.items(): + if isinstance(value, dict): + values.extend(find_values_with_key(value, key)) + if isinstance(value, list): + for item in value: + if isinstance(item, dict): + values.extend(find_values_with_key(item, key)) + if key == dic_key: + values.append(value) + return values + + +def get_schama_key(ref_str: str) -> str: + return ref_str.replace("#/components/schemas/", "") + + +def validate_schema_key(test_str: str) -> bool: + return re.fullmatch(regex, test_str) is not None + + +@shared_task(queue='dongtai-api-route-handler') +def api_route_gather_v2(agent_id, api_routes): + logger.info(_('API navigation log start '), ) + try: + agent = IastAgent.objects.get(pk=agent_id) + + schemas = api_routes['components']['schemas'] + + api_routes_list = [] + for path, path_info in api_routes['paths'].items(): + for method, api in path_info.items(): + ref_in_api = find_values_with_key(api, '$ref') + schema_keys = [get_schama_key(i) for i in ref_in_api] + extenal_schema_keys = [] + for key in schema_keys: + extenal_schema_keys.extend( + find_values_with_key(schemas[key], "$ref")) + extenal_schema_keys = [ + get_schama_key(i) for i in extenal_schema_keys + ] + keys = set(schema_keys + extenal_schema_keys) + new_dict = { + "paths": { + path: { + method: api + } + }, + "components": { + "schemas": {key: schemas[key] + for key in keys} + } + } + api_route = IastApiRouteV2( + path=path, + method=method, + from_where=FromWhereChoices.FROM_AGENT, + project_id=agent.bind_project_id, + project_version_id=agent.project_version_id, + api_info=new_dict, + ) + api_routes_list.append(api_route) + IastApiRouteV2.objects.bulk_create(api_routes_list, + ignore_conflicts=False) + except Exception as e: + logger.info( + _('API navigation log failed, why: {}').format(e), + exc_info=e, + ) diff --git a/dongtai_protocol/report/handler/saas_method_pool_handler.py b/dongtai_protocol/report/handler/saas_method_pool_handler.py index ff7fa19ab..f9e77da02 100644 --- a/dongtai_protocol/report/handler/saas_method_pool_handler.py +++ b/dongtai_protocol/report/handler/saas_method_pool_handler.py @@ -105,7 +105,7 @@ def save(self): """ headers = SaasMethodPoolHandler.parse_headers(self.http_req_header) #save_project_header(list(headers.keys()), self.agent_id) - add_new_api_route(self.agent, self.http_uri, self.http_method) + #add_new_api_route(self.agent, self.http_uri, self.http_method) import base64 params_dict = get_params_dict(base64.b64decode(self.http_req_header), self.http_req_data, diff --git a/test/apiserver/test_agent_apiroute_v2.py b/test/apiserver/test_agent_apiroute_v2.py new file mode 100644 index 000000000..2b8b7a55c --- /dev/null +++ b/test/apiserver/test_agent_apiroute_v2.py @@ -0,0 +1,16 @@ +from test.apiserver.test_agent_base import AgentTestCase +from dongtai_common.models.api_route_v2 import IastApiRouteV2 +from dongtai_common.models.agent import IastAgent +import json + + +class ApiRouteV2TestCase(AgentTestCase): + + def test_agent_api_upload(self): + with open('./test/integration/mockdata/api-report.json') as fp: + data = json.load(fp) + data['detail']['agentId'] = self.agent_id + res = self.agent_report(data, agentId=self.agent_id) + self.assertGreater( + IastApiRouteV2.objects.filter( + project_version__iastagent__pk=self.agent_id).count(), 0) diff --git a/test/integration/mockdata/api-report.json b/test/integration/mockdata/api-report.json new file mode 100644 index 000000000..d1cccee5f --- /dev/null +++ b/test/integration/mockdata/api-report.json @@ -0,0 +1 @@ +{"type":98,"detail":{"agentId":11,"framework":"spring mvc","apiData":{"components":{"schemas":{"HttpHeaders":{"name":"HttpHeaders","properties":{"CONTENT_LANGUAGE":{"type":"string"},"HOST":{"type":"string"},"IF_MODIFIED_SINCE":{"type":"string"},"FROM":{"type":"string"},"CACHE_CONTROL":{"type":"string"},"CONTENT_RANGE":{"type":"string"},"IF_RANGE":{"type":"string"},"CONTENT_TYPE":{"type":"string"},"PRAGMA":{"type":"string"},"VARY":{"type":"string"},"ALLOW":{"type":"string"},"IF_MATCH":{"type":"string"},"ACCEPT_ENCODING":{"type":"string"},"CONTENT_LOCATION":{"type":"string"},"TRANSFER_ENCODING":{"type":"string"},"USER_AGENT":{"type":"string"},"ACCEPT_RANGES":{"type":"string"},"ACCESS_CONTROL_ALLOW_HEADERS":{"type":"string"},"DATE_FORMATTER":{"$ref":"#/components/schemas/DateTimeFormatter"},"ACCESS_CONTROL_ALLOW_ORIGIN":{"type":"string"},"WARNING":{"type":"string"},"IF_NONE_MATCH":{"type":"string"},"GMT":{"$ref":"#/components/schemas/ZoneId"},"DATE":{"type":"string"},"ACCEPT":{"type":"string"},"CONTENT_LENGTH":{"type":"string"},"ETAG":{"type":"string"},"COOKIE":{"type":"string"},"RETRY_AFTER":{"type":"string"},"TRAILER":{"type":"string"},"ETAG_HEADER_VALUE_PATTERN":{"$ref":"#/components/schemas/Pattern"},"RANGE":{"type":"string"},"LOCATION":{"type":"string"},"CONTENT_DISPOSITION":{"type":"string"},"IF_UNMODIFIED_SINCE":{"type":"string"},"LAST_MODIFIED":{"type":"string"},"WWW_AUTHENTICATE":{"type":"string"},"ACCESS_CONTROL_ALLOW_CREDENTIALS":{"type":"string"},"serialVersionUID":{"format":"int64","type":"integer"},"ORIGIN":{"type":"string"},"SET_COOKIE2":{"type":"string"},"ACCESS_CONTROL_MAX_AGE":{"type":"string"},"EXPIRES":{"type":"string"},"SERVER":{"type":"string"},"LINK":{"type":"string"},"EMPTY":{"$ref":"#/components/schemas/HttpHeaders"},"CONTENT_ENCODING":{"type":"string"},"AGE":{"type":"string"},"DATE_PARSERS":{"items":{"$ref":"#/components/schemas/DateTimeFormatter"},"type":"array"},"headers":{"$ref":"#/components/schemas/MultiValueMap"},"EXPECT":{"type":"string"},"ACCESS_CONTROL_REQUEST_METHOD":{"type":"string"},"SET_COOKIE":{"type":"string"},"PROXY_AUTHORIZATION":{"type":"string"},"VIA":{"type":"string"},"ACCESS_CONTROL_REQUEST_HEADERS":{"type":"string"},"CONNECTION":{"type":"string"},"MAX_FORWARDS":{"type":"string"},"TE":{"type":"string"},"ACCEPT_LANGUAGE":{"type":"string"},"REFERER":{"type":"string"},"ACCEPT_CHARSET":{"type":"string"},"ACCESS_CONTROL_ALLOW_METHODS":{"type":"string"},"UPGRADE":{"type":"string"},"PROXY_AUTHENTICATE":{"type":"string"},"ACCESS_CONTROL_EXPOSE_HEADERS":{"type":"string"},"DECIMAL_FORMAT_SYMBOLS":{"$ref":"#/components/schemas/DecimalFormatSymbols"},"AUTHORIZATION":{"type":"string"}},"type":"object"},"CertificateFactorySpi":{"name":"CertificateFactorySpi","type":"object"},"Configuration":{"name":"Configuration","properties":{"EMPTY_CONFIGURATION":{"$ref":"#/components/schemas/Configuration"},"allConfigurations":{"items":{"type":"object"},"type":"array"},"nameToModule":{"$ref":"#/components/schemas/Map"},"targetPlatform":{"type":"string"},"graph":{"$ref":"#/components/schemas/Map"},"modules":{"items":{"type":"object"},"type":"array","uniqueItems":true},"parents":{"items":{"type":"object"},"type":"array"}},"type":"object"},"Node":{"name":"Node","properties":{"next":{"$ref":"#/components/schemas/Node"},"value":{"$ref":"#/components/schemas/Object"},"hash":{"format":"int32","type":"integer"},"key":{"$ref":"#/components/schemas/Object"}},"type":"object"},"CharPredicate":{"name":"CharPredicate","type":"object"},"DecimalStyle":{"name":"DecimalStyle","properties":{"negativeSign":{"type":"string"},"decimalSeparator":{"type":"string"},"zeroDigit":{"type":"string"},"CACHE":{"$ref":"#/components/schemas/ConcurrentMap"},"positiveSign":{"type":"string"},"STANDARD":{"$ref":"#/components/schemas/DecimalStyle"}},"type":"object"},"SoftReference":{"name":"SoftReference","properties":{"next":{"$ref":"#/components/schemas/Reference"},"discovered":{"$ref":"#/components/schemas/Reference"},"referent":{"$ref":"#/components/schemas/Object"},"processPendingActive":{"type":"boolean"},"clock":{"format":"int64","type":"integer"},"queue":{"$ref":"#/components/schemas/ReferenceQueue"},"timestamp":{"format":"int64","type":"integer"},"processPendingLock":{"$ref":"#/components/schemas/Object"}},"type":"object"},"ThreadLocal":{"name":"ThreadLocal","properties":{"HASH_INCREMENT":{"format":"int32","type":"integer"},"threadLocalHashCode":{"format":"int32","type":"integer"},"nextHashCode":{"$ref":"#/components/schemas/AtomicInteger"}},"type":"object"},"TypeVariable":{"name":"TypeVariable","type":"object"},"Currency":{"name":"Currency","properties":{"instances":{"$ref":"#/components/schemas/ConcurrentMap"},"NUMERIC_CODE_SHIFT":{"format":"int32","type":"integer"},"available":{"items":{"type":"object"},"type":"array","uniqueItems":true},"VALID_FORMAT_VERSION":{"format":"int32","type":"integer"},"otherCurrenciesList":{"items":{"type":"object"},"type":"array"},"INVALID_COUNTRY_ENTRY":{"format":"int32","type":"integer"},"SIMPLE_CASE_COUNTRY_MASK":{"format":"int32","type":"integer"},"SIMPLE_CASE_COUNTRY_MAX_DEFAULT_DIGITS":{"format":"int32","type":"integer"},"numericCode":{"format":"int32","type":"integer"},"serialVersionUID":{"format":"int64","type":"integer"},"COUNTRY_TYPE_MASK":{"format":"int32","type":"integer"},"SIMPLE_CASE_COUNTRY_DEFAULT_DIGITS_SHIFT":{"format":"int32","type":"integer"},"MAGIC_NUMBER":{"format":"int32","type":"integer"},"A_TO_Z":{"format":"int32","type":"integer"},"formatVersion":{"format":"int32","type":"integer"},"COUNTRY_WITHOUT_CURRENCY_ENTRY":{"format":"int32","type":"integer"},"DISPLAYNAME":{"format":"int32","type":"integer"},"specialCasesList":{"items":{"type":"object"},"type":"array"},"dataVersion":{"format":"int32","type":"integer"},"SIMPLE_CASE_COUNTRY_FINAL_CHAR_MASK":{"format":"int32","type":"integer"},"defaultFractionDigits":{"format":"int32","type":"integer"},"mainTable":{"items":{"format":"int32","type":"integer"},"type":"array"},"SPECIAL_CASE_COUNTRY_INDEX_DELTA":{"format":"int32","type":"integer"},"NUMERIC_CODE_MASK":{"format":"int32","type":"integer"},"SYMBOL":{"format":"int32","type":"integer"},"SPECIAL_CASE_COUNTRY_MASK":{"format":"int32","type":"integer"},"currencyCode":{"type":"string"},"SIMPLE_CASE_COUNTRY_DEFAULT_DIGITS_MASK":{"format":"int32","type":"integer"},"SPECIAL_CASE_COUNTRY_INDEX_MASK":{"format":"int32","type":"integer"}},"type":"object"},"Permission":{"name":"Permission","properties":{"serialVersionUID":{"format":"int64","type":"integer"},"name":{"type":"string"}},"type":"object"},"TemporalQuery":{"name":"TemporalQuery","type":"object"},"JavaLangAccess":{"name":"JavaLangAccess","type":"object"},"MethodAccessor":{"name":"MethodAccessor","type":"object"},"ServicesCatalog":{"name":"ServicesCatalog","properties":{"CLV":{"$ref":"#/components/schemas/ClassLoaderValue"},"map":{"$ref":"#/components/schemas/Map"}},"type":"object"},"Method":{"name":"Method","properties":{"genericInfo":{"$ref":"#/components/schemas/MethodRepository"},"hasRealParameterData":{"type":"boolean"},"declaredAnnotations":{"$ref":"#/components/schemas/Map"},"printStackPropertiesSet":{"type":"boolean"},"parameterTypes":{"items":{"$ref":"#/components/schemas/Class"},"type":"array"},"signature":{"type":"string"},"annotations":{"items":{"format":"int32","type":"integer"},"type":"array"},"securityCheckCache":{"$ref":"#/components/schemas/Object"},"printStackWhenAccessFails":{"type":"boolean"},"slot":{"format":"int32","type":"integer"},"modifiers":{"format":"int32","type":"integer"},"methodAccessor":{"$ref":"#/components/schemas/MethodAccessor"},"ACCESS_PERMISSION":{"$ref":"#/components/schemas/Permission"},"exceptionTypes":{"items":{"$ref":"#/components/schemas/Class"},"type":"array"},"annotationDefault":{"items":{"format":"int32","type":"integer"},"type":"array"},"root":{"$ref":"#/components/schemas/Method"},"name":{"type":"string"},"parameterAnnotations":{"items":{"format":"int32","type":"integer"},"type":"array"},"override":{"type":"boolean"},"reflectionFactory":{"$ref":"#/components/schemas/ReflectionFactory"},"clazz":{"$ref":"#/components/schemas/Class"},"parameters":{"items":{"$ref":"#/components/schemas/Parameter"},"type":"array"},"returnType":{"$ref":"#/components/schemas/Class"}},"type":"object"},"CodeSource":{"name":"CodeSource","properties":{"signers":{"items":{"$ref":"#/components/schemas/CodeSigner"},"type":"array"},"factory":{"$ref":"#/components/schemas/CertificateFactory"},"serialVersionUID":{"format":"int64","type":"integer"},"location":{"$ref":"#/components/schemas/URL"},"locationNoFragString":{"type":"string"},"certs":{"items":{"$ref":"#/components/schemas/Certificate"},"type":"array"},"sp":{"$ref":"#/components/schemas/SocketPermission"}},"type":"object"},"URLStreamHandler":{"name":"URLStreamHandler","type":"object"},"TimeZone":{"name":"TimeZone","properties":{"ONE_HOUR":{"format":"int32","type":"integer"},"GMT_ID":{"type":"string"},"$assertionsDisabled":{"type":"boolean"},"GMT_ID_LENGTH":{"format":"int32","type":"integer"},"SHORT":{"format":"int32","type":"integer"},"serialVersionUID":{"format":"int64","type":"integer"},"defaultTimeZone":{"$ref":"#/components/schemas/TimeZone"},"mainAppContextDefault":{"$ref":"#/components/schemas/TimeZone"},"NO_TIMEZONE":{"$ref":"#/components/schemas/TimeZone"},"zoneId":{"$ref":"#/components/schemas/ZoneId"},"ID":{"type":"string"},"ONE_DAY":{"format":"int32","type":"integer"},"ONE_MINUTE":{"format":"int32","type":"integer"},"LONG":{"format":"int32","type":"integer"}},"type":"object"},"AtomicInteger":{"name":"AtomicInteger","properties":{"serialVersionUID":{"format":"int64","type":"integer"},"U":{"$ref":"#/components/schemas/Unsafe"},"VALUE":{"format":"int64","type":"integer"},"value":{"format":"int32","type":"integer"}},"type":"object"},"Era":{"name":"Era","properties":{"localTime":{"type":"boolean"},"name":{"type":"string"},"abbr":{"type":"string"},"sinceDate":{"$ref":"#/components/schemas/CalendarDate"},"hash":{"format":"int32","type":"integer"},"since":{"format":"int64","type":"integer"}},"type":"object"},"Object":{"name":"Object","type":"object"},"ModuleLayer":{"name":"ModuleLayer","properties":{"cf":{"$ref":"#/components/schemas/Configuration"},"servicesCatalog":{"$ref":"#/components/schemas/ServicesCatalog"},"EMPTY_LAYER":{"$ref":"#/components/schemas/ModuleLayer"},"nameToModule":{"$ref":"#/components/schemas/Map"},"CLV":{"$ref":"#/components/schemas/ClassLoaderValue"},"allLayers":{"items":{"type":"object"},"type":"array"},"modules":{"items":{"type":"object"},"type":"array","uniqueItems":true},"parents":{"items":{"type":"object"},"type":"array"}},"type":"object"},"Module":{"name":"Module","properties":{"EVERYONE_MODULE":{"$ref":"#/components/schemas/Module"},"loader":{"$ref":"#/components/schemas/ClassLoader"},"openPackages":{"$ref":"#/components/schemas/Map"},"moduleInfoClass":{"$ref":"#/components/schemas/Class"},"reads":{"items":{"type":"object"},"type":"array","uniqueItems":true},"$assertionsDisabled":{"type":"boolean"},"descriptor":{"$ref":"#/components/schemas/ModuleDescriptor"},"exportedPackages":{"$ref":"#/components/schemas/Map"},"reflectivelyExports":{"$ref":"#/components/schemas/WeakPairMap"},"layer":{"$ref":"#/components/schemas/ModuleLayer"},"ALL_UNNAMED_MODULE":{"$ref":"#/components/schemas/Module"},"reflectivelyUses":{"$ref":"#/components/schemas/WeakPairMap"},"reflectivelyReads":{"$ref":"#/components/schemas/WeakPairMap"},"name":{"type":"string"},"EVERYONE_SET":{"items":{"type":"object"},"type":"array","uniqueItems":true},"ALL_UNNAMED_MODULE_SET":{"items":{"type":"object"},"type":"array","uniqueItems":true}},"type":"object"},"InetAddress":{"name":"InetAddress","properties":{"cache":{"$ref":"#/components/schemas/ConcurrentMap"},"PREFER_IPV6_VALUE":{"format":"int32","type":"integer"},"IPv6":{"format":"int32","type":"integer"},"nameService":{"$ref":"#/components/schemas/NameService"},"canonicalHostName":{"type":"string"},"PREFER_IPV4_VALUE":{"format":"int32","type":"integer"},"IPv4":{"format":"int32","type":"integer"},"PREFER_SYSTEM_VALUE":{"format":"int32","type":"integer"},"holder":{"$ref":"#/components/schemas/InetAddressHolder"},"cachedLocalHost":{"$ref":"#/components/schemas/CachedLocalHost"},"FIELDS_OFFSET":{"format":"int64","type":"integer"},"serialPersistentFields":{"items":{"$ref":"#/components/schemas/ObjectStreamField"},"type":"array"},"impl":{"$ref":"#/components/schemas/InetAddressImpl"},"serialVersionUID":{"format":"int64","type":"integer"},"preferIPv6Address":{"format":"int32","type":"integer"},"UNSAFE":{"$ref":"#/components/schemas/Unsafe"},"expirySet":{"items":{"type":"object"},"type":"array","uniqueItems":true}},"type":"object"},"Map":{"name":"Map","type":"object"},"Key":{"name":"Key","type":"object"},"LangReflectAccess":{"name":"LangReflectAccess","type":"object"},"GroupTail":{"name":"GroupTail","properties":{"next":{"$ref":"#/components/schemas/Node"},"localIndex":{"format":"int32","type":"integer"},"groupIndex":{"format":"int32","type":"integer"}},"type":"object"},"CertificateFactory":{"name":"CertificateFactory","properties":{"provider":{"$ref":"#/components/schemas/Provider"},"type":{"type":"string"},"certFacSpi":{"$ref":"#/components/schemas/CertificateFactorySpi"}},"type":"object"},"Timestamp":{"name":"Timestamp","properties":{"myhash":{"format":"int32","type":"integer"},"serialVersionUID":{"format":"int64","type":"integer"},"signerCertPath":{"$ref":"#/components/schemas/CertPath"},"timestamp":{"$ref":"#/components/schemas/Date"}},"type":"object"},"URL":{"name":"URL","properties":{"userInfo":{"type":"string"},"handler":{"$ref":"#/components/schemas/URLStreamHandler"},"factory":{"$ref":"#/components/schemas/URLStreamHandlerFactory"},"query":{"type":"string"},"protocolPathProp":{"type":"string"},"serialPersistentFields":{"items":{"$ref":"#/components/schemas/ObjectStreamField"},"type":"array"},"path":{"type":"string"},"serialVersionUID":{"format":"int64","type":"integer"},"protocol":{"type":"string"},"ref":{"type":"string"},"file":{"type":"string"},"port":{"format":"int32","type":"integer"},"hashCode":{"format":"int32","type":"integer"},"defaultFactory":{"$ref":"#/components/schemas/URLStreamHandlerFactory"},"handlers":{"$ref":"#/components/schemas/Hashtable"},"authority":{"type":"string"},"host":{"type":"string"},"BUILTIN_HANDLERS_PREFIX":{"type":"string"},"hostAddress":{"$ref":"#/components/schemas/InetAddress"},"tempState":{"$ref":"#/components/schemas/UrlDeserializedState"},"gate":{"$ref":"#/components/schemas/ThreadLocal"},"streamHandlerLock":{"$ref":"#/components/schemas/Object"}},"type":"object"},"AnnotationType":{"name":"AnnotationType","properties":{"inherited":{"type":"boolean"},"members":{"$ref":"#/components/schemas/Map"},"memberDefaults":{"$ref":"#/components/schemas/Map"},"memberTypes":{"$ref":"#/components/schemas/Map"},"$assertionsDisabled":{"type":"boolean"},"retention":{"enums":["SOURCE","CLASS","RUNTIME"],"type":"string"}},"type":"object"},"Entry":{"name":"Entry","properties":{"next":{"$ref":"#/components/schemas/Entry"},"value":{"$ref":"#/components/schemas/Object"},"hash":{"format":"int32","type":"integer"},"key":{"$ref":"#/components/schemas/Object"}},"type":"object"},"WeakPairMap":{"name":"WeakPairMap","properties":{"map":{"$ref":"#/components/schemas/ConcurrentHashMap"},"queue":{"$ref":"#/components/schemas/ReferenceQueue"}},"type":"object"},"CertPath":{"name":"CertPath","properties":{"serialVersionUID":{"format":"int64","type":"integer"},"type":{"type":"string"}},"type":"object"},"CompositePrinterParser":{"name":"CompositePrinterParser","properties":{"optional":{"type":"boolean"},"printerParsers":{"items":{"$ref":"#/components/schemas/DateTimePrinterParser"},"type":"array"}},"type":"object"},"ConcurrentHashMap":{"name":"ConcurrentHashMap","properties":{"UNTREEIFY_THRESHOLD":{"format":"int32","type":"integer"},"TREEIFY_THRESHOLD":{"format":"int32","type":"integer"},"cellsBusy":{"format":"int32","type":"integer"},"values":{"$ref":"#/components/schemas/Collection"},"RESIZE_STAMP_SHIFT":{"format":"int32","type":"integer"},"DEFAULT_CONCURRENCY_LEVEL":{"format":"int32","type":"integer"},"HASH_BITS":{"format":"int32","type":"integer"},"baseCount":{"format":"int64","type":"integer"},"SIZECTL":{"format":"int64","type":"integer"},"CELLVALUE":{"format":"int64","type":"integer"},"serialVersionUID":{"format":"int64","type":"integer"},"U":{"$ref":"#/components/schemas/Unsafe"},"TRANSFERINDEX":{"format":"int64","type":"integer"},"transferIndex":{"format":"int32","type":"integer"},"LOAD_FACTOR":{"format":"float","type":"number"},"keySet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"DEFAULT_CAPACITY":{"format":"int32","type":"integer"},"NCPU":{"format":"int32","type":"integer"},"table":{"items":{"$ref":"#/components/schemas/Node"},"type":"array"},"ASHIFT":{"format":"int32","type":"integer"},"ABASE":{"format":"int32","type":"integer"},"sizeCtl":{"format":"int32","type":"integer"},"entrySet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"RESERVED":{"format":"int32","type":"integer"},"TREEBIN":{"format":"int32","type":"integer"},"MIN_TRANSFER_STRIDE":{"format":"int32","type":"integer"},"counterCells":{"items":{"$ref":"#/components/schemas/CounterCell"},"type":"array"},"serialPersistentFields":{"items":{"$ref":"#/components/schemas/ObjectStreamField"},"type":"array"},"MOVED":{"format":"int32","type":"integer"},"nextTable":{"items":{"$ref":"#/components/schemas/Node"},"type":"array"},"MAX_ARRAY_SIZE":{"format":"int32","type":"integer"},"MAX_RESIZERS":{"format":"int32","type":"integer"},"RESIZE_STAMP_BITS":{"format":"int32","type":"integer"},"BASECOUNT":{"format":"int64","type":"integer"},"MAXIMUM_CAPACITY":{"format":"int32","type":"integer"},"MIN_TREEIFY_CAPACITY":{"format":"int32","type":"integer"},"CELLSBUSY":{"format":"int64","type":"integer"}},"type":"object"},"MultiValueMap":{"name":"MultiValueMap","type":"object"},"CachedLocalHost":{"name":"CachedLocalHost","properties":{"host":{"type":"string"},"expiryTime":{"format":"int64","type":"integer"},"addr":{"$ref":"#/components/schemas/InetAddress"}},"type":"object"},"Pattern":{"name":"Pattern","properties":{"cursor":{"format":"int32","type":"integer"},"ALL_FLAGS":{"format":"int32","type":"integer"},"pattern":{"type":"string"},"flags":{"format":"int32","type":"integer"},"COMMENTS":{"format":"int32","type":"integer"},"namedGroups":{"$ref":"#/components/schemas/Map"},"localCount":{"format":"int32","type":"integer"},"groupNodes":{"items":{"$ref":"#/components/schemas/GroupHead"},"type":"array"},"hasGroupRef":{"type":"boolean"},"lastAccept":{"$ref":"#/components/schemas/Node"},"predicate":{"$ref":"#/components/schemas/CharPredicate"},"serialVersionUID":{"format":"int64","type":"integer"},"compiled":{"type":"boolean"},"normalizedPattern":{"type":"string"},"CASE_INSENSITIVE":{"format":"int32","type":"integer"},"root":{"$ref":"#/components/schemas/Node"},"buffer":{"items":{"format":"int32","type":"integer"},"type":"array"},"matchRoot":{"$ref":"#/components/schemas/Node"},"capturingGroupCount":{"format":"int32","type":"integer"},"lookbehindEnd":{"$ref":"#/components/schemas/Node"},"temp":{"items":{"format":"int32","type":"integer"},"type":"array"},"LITERAL":{"format":"int32","type":"integer"},"UNICODE_CHARACTER_CLASS":{"format":"int32","type":"integer"},"CANON_EQ":{"format":"int32","type":"integer"},"patternLength":{"format":"int32","type":"integer"},"localTCNCount":{"format":"int32","type":"integer"},"hasSupplementary":{"type":"boolean"},"$assertionsDisabled":{"type":"boolean"},"topClosureNodes":{"items":{"type":"object"},"type":"array"},"MAX_REPS":{"format":"int32","type":"integer"},"UNIX_LINES":{"format":"int32","type":"integer"},"accept":{"$ref":"#/components/schemas/Node"},"DOTALL":{"format":"int32","type":"integer"},"UNICODE_CASE":{"format":"int32","type":"integer"},"MULTILINE":{"format":"int32","type":"integer"}},"type":"object"},"UrlDeserializedState":{"name":"UrlDeserializedState","properties":{"protocol":{"type":"string"},"ref":{"type":"string"},"file":{"type":"string"},"port":{"format":"int32","type":"integer"},"hashCode":{"format":"int32","type":"integer"},"authority":{"type":"string"},"host":{"type":"string"}},"type":"object"},"DecimalFormatSymbols":{"name":"DecimalFormatSymbols","properties":{"exponential":{"type":"string"},"minusSign":{"type":"string"},"currencyInitialized":{"type":"boolean"},"decimalSeparator":{"type":"string"},"intlCurrencySymbol":{"type":"string"},"perMill":{"type":"string"},"currencySymbol":{"type":"string"},"monetarySeparator":{"type":"string"},"groupingSeparator":{"type":"string"},"locale":{"$ref":"#/components/schemas/Locale"},"percent":{"type":"string"},"currentSerialVersion":{"format":"int32","type":"integer"},"serialVersionUID":{"format":"int64","type":"integer"},"zeroDigit":{"type":"string"},"exponentialSeparator":{"type":"string"},"infinity":{"type":"string"},"NaN":{"type":"string"},"currency":{"$ref":"#/components/schemas/Currency"},"serialVersionOnStream":{"format":"int32","type":"integer"},"digit":{"type":"string"},"patternSeparator":{"type":"string"}},"type":"object"},"Hashtable":{"name":"Hashtable","properties":{"entrySet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"values":{"$ref":"#/components/schemas/Collection"},"count":{"format":"int32","type":"integer"},"threshold":{"format":"int32","type":"integer"},"modCount":{"format":"int32","type":"integer"},"serialVersionUID":{"format":"int64","type":"integer"},"MAX_ARRAY_SIZE":{"format":"int32","type":"integer"},"loadFactor":{"format":"float","type":"number"},"KEYS":{"format":"int32","type":"integer"},"VALUES":{"format":"int32","type":"integer"},"keySet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"table":{"items":{"$ref":"#/components/schemas/Entry"},"type":"array"},"ENTRIES":{"format":"int32","type":"integer"}},"type":"object"},"Constructor":{"name":"Constructor","properties":{"genericInfo":{"$ref":"#/components/schemas/ConstructorRepository"},"hasRealParameterData":{"type":"boolean"},"declaredAnnotations":{"$ref":"#/components/schemas/Map"},"printStackPropertiesSet":{"type":"boolean"},"parameterTypes":{"items":{"$ref":"#/components/schemas/Class"},"type":"array"},"signature":{"type":"string"},"annotations":{"items":{"format":"int32","type":"integer"},"type":"array"},"securityCheckCache":{"$ref":"#/components/schemas/Object"},"printStackWhenAccessFails":{"type":"boolean"},"slot":{"format":"int32","type":"integer"},"constructorAccessor":{"$ref":"#/components/schemas/ConstructorAccessor"},"modifiers":{"format":"int32","type":"integer"},"ACCESS_PERMISSION":{"$ref":"#/components/schemas/Permission"},"exceptionTypes":{"items":{"$ref":"#/components/schemas/Class"},"type":"array"},"root":{"$ref":"#/components/schemas/Constructor"},"parameterAnnotations":{"items":{"format":"int32","type":"integer"},"type":"array"},"override":{"type":"boolean"},"reflectionFactory":{"$ref":"#/components/schemas/ReflectionFactory"},"clazz":{"$ref":"#/components/schemas/Class"},"parameters":{"items":{"$ref":"#/components/schemas/Parameter"},"type":"array"}},"type":"object"},"Reference":{"name":"Reference","properties":{"next":{"$ref":"#/components/schemas/Reference"},"discovered":{"$ref":"#/components/schemas/Reference"},"referent":{"$ref":"#/components/schemas/Object"},"processPendingActive":{"type":"boolean"},"queue":{"$ref":"#/components/schemas/ReferenceQueue"},"processPendingLock":{"$ref":"#/components/schemas/Object"}},"type":"object"},"SocketPermission":{"name":"SocketPermission","properties":{"defaultDeny":{"type":"boolean"},"ALL":{"format":"int32","type":"integer"},"addresses":{"items":{"$ref":"#/components/schemas/InetAddress"},"type":"array"},"cname":{"type":"string"},"init_with_ip":{"type":"boolean"},"CONNECT":{"format":"int32","type":"integer"},"trustNameService":{"type":"boolean"},"wildcard":{"type":"boolean"},"serialVersionUID":{"format":"int64","type":"integer"},"PRIV_PORT_MAX":{"format":"int32","type":"integer"},"hostname":{"type":"string"},"PORT_MIN":{"format":"int32","type":"integer"},"hdomain":{"type":"string"},"LISTEN":{"format":"int32","type":"integer"},"cdomain":{"type":"string"},"NONE":{"format":"int32","type":"integer"},"DEF_EPH_LOW":{"format":"int32","type":"integer"},"mask":{"format":"int32","type":"integer"},"portrange":{"items":{"format":"int32","type":"integer"},"type":"array"},"debug":{"$ref":"#/components/schemas/Debug"},"PORT_MAX":{"format":"int32","type":"integer"},"debugInit":{"type":"boolean"},"ACCEPT":{"format":"int32","type":"integer"},"untrusted":{"type":"boolean"},"trusted":{"type":"boolean"},"RESOLVE":{"format":"int32","type":"integer"},"invalid":{"type":"boolean"},"name":{"type":"string"},"actions":{"type":"string"}},"type":"object"},"ModelMap":{"name":"ModelMap","properties":{"UNTREEIFY_THRESHOLD":{"format":"int32","type":"integer"},"TREEIFY_THRESHOLD":{"format":"int32","type":"integer"},"entrySet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"tail":{"$ref":"#/components/schemas/Entry"},"values":{"$ref":"#/components/schemas/Collection"},"DEFAULT_LOAD_FACTOR":{"format":"float","type":"number"},"threshold":{"format":"int32","type":"integer"},"DEFAULT_INITIAL_CAPACITY":{"format":"int32","type":"integer"},"head":{"$ref":"#/components/schemas/Entry"},"modCount":{"format":"int32","type":"integer"},"serialVersionUID":{"format":"int64","type":"integer"},"size":{"format":"int32","type":"integer"},"loadFactor":{"format":"float","type":"number"},"MAXIMUM_CAPACITY":{"format":"int32","type":"integer"},"keySet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"accessOrder":{"type":"boolean"},"MIN_TREEIFY_CAPACITY":{"format":"int32","type":"integer"},"table":{"items":{"$ref":"#/components/schemas/Node"},"type":"array"}},"type":"object"},"Lock":{"name":"Lock","type":"object"},"NameService":{"name":"NameService","type":"object"},"Type":{"name":"Type","type":"object"},"ResponseEntity":{"name":"ResponseEntity","properties":{"headers":{"$ref":"#/components/schemas/HttpHeaders"},"body":{"$ref":"#/components/schemas/Object"},"EMPTY":{"$ref":"#/components/schemas/HttpEntity"},"status":{"$ref":"#/components/schemas/Object"}},"type":"object"},"Parameter":{"name":"Parameter","properties":{"declaredAnnotations":{"$ref":"#/components/schemas/Map"},"parameterTypeCache":{"$ref":"#/components/schemas/Type"},"parameterClassCache":{"$ref":"#/components/schemas/Class"},"name":{"type":"string"},"index":{"format":"int32","type":"integer"},"modifiers":{"format":"int32","type":"integer"},"executable":{"$ref":"#/components/schemas/Executable"}},"type":"object"},"Principal":{"name":"Principal","type":"object"},"BaseLocale":{"name":"BaseLocale","properties":{"variant":{"type":"string"},"language":{"type":"string"},"CACHE":{"$ref":"#/components/schemas/Cache"},"region":{"type":"string"},"script":{"type":"string"},"hash":{"format":"int32","type":"integer"},"SEP":{"type":"string"}},"type":"object"},"Locale":{"name":"Locale","properties":{"PRC":{"$ref":"#/components/schemas/Locale"},"LOCALECACHE":{"$ref":"#/components/schemas/Cache"},"CANADA":{"$ref":"#/components/schemas/Locale"},"ROOT":{"$ref":"#/components/schemas/Locale"},"TAIWAN":{"$ref":"#/components/schemas/Locale"},"baseLocale":{"$ref":"#/components/schemas/BaseLocale"},"CHINESE":{"$ref":"#/components/schemas/Locale"},"DISPLAY_LANGUAGE":{"format":"int32","type":"integer"},"languageTag":{"type":"string"},"KOREA":{"$ref":"#/components/schemas/Locale"},"hashCodeValue":{"format":"int32","type":"integer"},"FRENCH":{"$ref":"#/components/schemas/Locale"},"FRANCE":{"$ref":"#/components/schemas/Locale"},"TRADITIONAL_CHINESE":{"$ref":"#/components/schemas/Locale"},"serialVersionUID":{"format":"int64","type":"integer"},"GERMANY":{"$ref":"#/components/schemas/Locale"},"DISPLAY_COUNTRY":{"format":"int32","type":"integer"},"ITALIAN":{"$ref":"#/components/schemas/Locale"},"PRIVATE_USE_EXTENSION":{"type":"string"},"defaultDisplayLocale":{"$ref":"#/components/schemas/Locale"},"UK":{"$ref":"#/components/schemas/Locale"},"JAPAN":{"$ref":"#/components/schemas/Locale"},"UNICODE_LOCALE_EXTENSION":{"type":"string"},"SIMPLIFIED_CHINESE":{"$ref":"#/components/schemas/Locale"},"DISPLAY_VARIANT":{"format":"int32","type":"integer"},"GERMAN":{"$ref":"#/components/schemas/Locale"},"US":{"$ref":"#/components/schemas/Locale"},"ENGLISH":{"$ref":"#/components/schemas/Locale"},"CHINA":{"$ref":"#/components/schemas/Locale"},"ITALY":{"$ref":"#/components/schemas/Locale"},"defaultFormatLocale":{"$ref":"#/components/schemas/Locale"},"$assertionsDisabled":{"type":"boolean"},"isoCountries":{"items":{"type":"string"},"type":"array"},"CANADA_FRENCH":{"$ref":"#/components/schemas/Locale"},"defaultLocale":{"$ref":"#/components/schemas/Locale"},"serialPersistentFields":{"items":{"$ref":"#/components/schemas/ObjectStreamField"},"type":"array"},"localeExtensions":{"$ref":"#/components/schemas/LocaleExtensions"},"isoLanguages":{"items":{"type":"string"},"type":"array"},"JAPANESE":{"$ref":"#/components/schemas/Locale"},"DISPLAY_SCRIPT":{"format":"int32","type":"integer"},"KOREAN":{"$ref":"#/components/schemas/Locale"}},"type":"object"},"ProtectionDomain":{"name":"ProtectionDomain","properties":{"staticPermissions":{"type":"boolean"},"hasAllPerm":{"type":"boolean"},"codesource":{"$ref":"#/components/schemas/CodeSource"},"permissions":{"$ref":"#/components/schemas/PermissionCollection"},"classloader":{"$ref":"#/components/schemas/ClassLoader"},"filePermCompatInPD":{"type":"boolean"},"principals":{"items":{"$ref":"#/components/schemas/Principal"},"type":"array"},"key":{"$ref":"#/components/schemas/Key"}},"type":"object"},"ConstructorRepository":{"name":"ConstructorRepository","properties":{"factory":{"$ref":"#/components/schemas/GenericsFactory"},"exceptionTypes":{"items":{"$ref":"#/components/schemas/Type"},"type":"array"},"parameterTypes":{"items":{"$ref":"#/components/schemas/Type"},"type":"array"},"tree":{"$ref":"#/components/schemas/Tree"},"typeParameters":{"items":{"$ref":"#/components/schemas/TypeVariable"},"type":"array"}},"type":"object"},"ClassRepository":{"name":"ClassRepository","properties":{"factory":{"$ref":"#/components/schemas/GenericsFactory"},"superclass":{"$ref":"#/components/schemas/Type"},"tree":{"$ref":"#/components/schemas/Tree"},"NONE":{"$ref":"#/components/schemas/ClassRepository"},"superInterfaces":{"items":{"$ref":"#/components/schemas/Type"},"type":"array"},"typeParameters":{"items":{"$ref":"#/components/schemas/TypeVariable"},"type":"array"}},"type":"object"},"ServiceKey":{"name":"ServiceKey","properties":{"originalAlgorithm":{"type":"string"},"type":{"type":"string"},"algorithm":{"type":"string"}},"type":"object"},"Unsafe":{"name":"Unsafe","properties":{"ARRAY_BOOLEAN_BASE_OFFSET":{"format":"int32","type":"integer"},"ARRAY_CHAR_BASE_OFFSET":{"format":"int32","type":"integer"},"BE":{"type":"boolean"},"ARRAY_SHORT_BASE_OFFSET":{"format":"int32","type":"integer"},"ARRAY_INT_BASE_OFFSET":{"format":"int32","type":"integer"},"INVALID_FIELD_OFFSET":{"format":"int32","type":"integer"},"ARRAY_OBJECT_INDEX_SCALE":{"format":"int32","type":"integer"},"ADDRESS_SIZE":{"format":"int32","type":"integer"},"ARRAY_LONG_INDEX_SCALE":{"format":"int32","type":"integer"},"ARRAY_FLOAT_INDEX_SCALE":{"format":"int32","type":"integer"},"ARRAY_DOUBLE_INDEX_SCALE":{"format":"int32","type":"integer"},"unalignedAccess":{"type":"boolean"},"ARRAY_FLOAT_BASE_OFFSET":{"format":"int32","type":"integer"},"ARRAY_DOUBLE_BASE_OFFSET":{"format":"int32","type":"integer"},"theUnsafe":{"$ref":"#/components/schemas/Unsafe"},"ARRAY_BOOLEAN_INDEX_SCALE":{"format":"int32","type":"integer"},"ARRAY_CHAR_INDEX_SCALE":{"format":"int32","type":"integer"},"ARRAY_INT_INDEX_SCALE":{"format":"int32","type":"integer"},"ARRAY_BYTE_INDEX_SCALE":{"format":"int32","type":"integer"},"ARRAY_LONG_BASE_OFFSET":{"format":"int32","type":"integer"},"ARRAY_OBJECT_BASE_OFFSET":{"format":"int32","type":"integer"},"ARRAY_SHORT_INDEX_SCALE":{"format":"int32","type":"integer"},"ARRAY_BYTE_BASE_OFFSET":{"format":"int32","type":"integer"}},"type":"object"},"CalendarDate":{"name":"CalendarDate","properties":{"zoneOffset":{"format":"int32","type":"integer"},"hours":{"format":"int32","type":"integer"},"zoneinfo":{"$ref":"#/components/schemas/TimeZone"},"year":{"format":"int32","type":"integer"},"minutes":{"format":"int32","type":"integer"},"normalized":{"type":"boolean"},"forceStandardTime":{"type":"boolean"},"FIELD_UNDEFINED":{"format":"int32","type":"integer"},"leapYear":{"type":"boolean"},"locale":{"$ref":"#/components/schemas/Locale"},"fraction":{"format":"int64","type":"integer"},"TIME_UNDEFINED":{"format":"int64","type":"integer"},"seconds":{"format":"int32","type":"integer"},"dayOfWeek":{"format":"int32","type":"integer"},"month":{"format":"int32","type":"integer"},"daylightSaving":{"format":"int32","type":"integer"},"era":{"$ref":"#/components/schemas/Era"},"dayOfMonth":{"format":"int32","type":"integer"},"millis":{"format":"int32","type":"integer"}},"type":"object"},"CounterCell":{"name":"CounterCell","properties":{"value":{"format":"int64","type":"integer"}},"type":"object"},"Version":{"name":"Version","properties":{"sequence":{"items":{"type":"object"},"type":"array"},"pre":{"items":{"type":"object"},"type":"array"},"build":{"items":{"type":"object"},"type":"array"},"version":{"type":"string"}},"type":"object"},"ReflectionFactory":{"name":"ReflectionFactory","properties":{"initted":{"type":"boolean"},"inflationThreshold":{"format":"int32","type":"integer"},"soleInstance":{"$ref":"#/components/schemas/ReflectionFactory"},"reflectionFactoryAccessPerm":{"$ref":"#/components/schemas/Permission"},"langReflectAccess":{"$ref":"#/components/schemas/LangReflectAccess"},"hasStaticInitializerMethod":{"$ref":"#/components/schemas/Method"},"noInflation":{"type":"boolean"}},"type":"object"},"Tree":{"name":"Tree","type":"object"},"Executable":{"name":"Executable","properties":{"hasRealParameterData":{"type":"boolean"},"declaredAnnotations":{"$ref":"#/components/schemas/Map"},"printStackPropertiesSet":{"type":"boolean"},"securityCheckCache":{"$ref":"#/components/schemas/Object"},"printStackWhenAccessFails":{"type":"boolean"},"override":{"type":"boolean"},"reflectionFactory":{"$ref":"#/components/schemas/ReflectionFactory"},"parameters":{"items":{"$ref":"#/components/schemas/Parameter"},"type":"array"},"ACCESS_PERMISSION":{"$ref":"#/components/schemas/Permission"}},"type":"object"},"Foo":{"name":"Foo","properties":{"l1":{"format":"int64","type":"integer"},"l2":{"items":{"format":"int64","type":"integer"},"type":"array"},"foo":{"$ref":"#/components/schemas/Foo"},"f1":{"format":"float","type":"number"},"f2":{"items":{"format":"float","type":"number"},"type":"array"},"d1":{"format":"double","type":"number"},"d2":{"items":{"format":"double","type":"number"},"type":"array"},"b1":{"type":"boolean"},"b2":{"items":{"type":"boolean"},"type":"array"},"ss1":{"items":{"type":"object"},"type":"array","uniqueItems":true},"bar2":{"items":{"$ref":"#/components/schemas/Bar"},"type":"array"},"bar":{"$ref":"#/components/schemas/Bar"},"foo2":{"items":{"$ref":"#/components/schemas/Foo"},"type":"array"},"s1":{"format":"int32","type":"integer"},"s2":{"items":{"format":"int32","type":"integer"},"type":"array"},"m1":{"$ref":"#/components/schemas/Map"},"k1":{"type":"string"},"k2":{"items":{"type":"string"},"type":"array"},"k3":{"items":{"type":"object"},"type":"array"},"i1":{"format":"int32","type":"integer"},"i2":{"items":{"format":"int32","type":"integer"},"type":"array"},"enumnumnum":{"enums":["A","B","C"],"type":"string"},"c1":{"type":"string"},"c2":{"items":{"type":"string"},"type":"array"},"list1":{"items":{"type":"object"},"type":"array"},"list4":{"items":{"type":"object"},"type":"array"},"list3":{"items":{"type":"object"},"type":"array"},"list2":{"items":{"type":"object"},"type":"array"}},"type":"object"},"ConcurrentMap":{"name":"ConcurrentMap","type":"object"},"ValuesView":{"name":"ValuesView","properties":{"serialVersionUID":{"format":"int64","type":"integer"},"OOME_MSG":{"type":"string"},"map":{"$ref":"#/components/schemas/ConcurrentHashMap"}},"type":"object"},"MethodRepository":{"name":"MethodRepository","properties":{"factory":{"$ref":"#/components/schemas/GenericsFactory"},"exceptionTypes":{"items":{"$ref":"#/components/schemas/Type"},"type":"array"},"parameterTypes":{"items":{"$ref":"#/components/schemas/Type"},"type":"array"},"tree":{"$ref":"#/components/schemas/Tree"},"typeParameters":{"items":{"$ref":"#/components/schemas/TypeVariable"},"type":"array"},"returnType":{"$ref":"#/components/schemas/Type"}},"type":"object"},"ObjectStreamField":{"name":"ObjectStreamField","properties":{"field":{"$ref":"#/components/schemas/Field"},"offset":{"format":"int32","type":"integer"},"signature":{"type":"string"},"unshared":{"type":"boolean"},"name":{"type":"string"},"type":{"$ref":"#/components/schemas/Class"}},"type":"object"},"GenericsFactory":{"name":"GenericsFactory","type":"object"},"Collection":{"name":"Collection","type":"object"},"Class":{"name":"Class","properties":{"genericInfo":{"$ref":"#/components/schemas/ClassRepository"},"annotationData":{"$ref":"#/components/schemas/AnnotationData"},"ENUM":{"format":"int32","type":"integer"},"componentType":{"$ref":"#/components/schemas/Class"},"enumConstantDirectory":{"$ref":"#/components/schemas/Map"},"classRedefinedCount":{"format":"int32","type":"integer"},"cachedConstructor":{"$ref":"#/components/schemas/Constructor"},"module":{"$ref":"#/components/schemas/Module"},"SYNTHETIC":{"format":"int32","type":"integer"},"annotationType":{"$ref":"#/components/schemas/AnnotationType"},"newInstanceCallerCache":{"$ref":"#/components/schemas/Class"},"reflectionData":{"$ref":"#/components/schemas/SoftReference"},"classValueMap":{"$ref":"#/components/schemas/ClassValueMap"},"serialPersistentFields":{"items":{"$ref":"#/components/schemas/ObjectStreamField"},"type":"array"},"serialVersionUID":{"format":"int64","type":"integer"},"ANNOTATION":{"format":"int32","type":"integer"},"enumConstants":{"items":{"$ref":"#/components/schemas/Object"},"type":"array"},"name":{"type":"string"},"packageName":{"type":"string"},"reflectionFactory":{"$ref":"#/components/schemas/ReflectionFactory"},"allPermDomain":{"$ref":"#/components/schemas/ProtectionDomain"},"EMPTY_CLASS_ARRAY":{"items":{"$ref":"#/components/schemas/Class"},"type":"array"}},"type":"object"},"InetAddressImpl":{"name":"InetAddressImpl","type":"object"},"DateTimeFormatter":{"name":"DateTimeFormatter","properties":{"printerParser":{"$ref":"#/components/schemas/CompositePrinterParser"},"ISO_OFFSET_TIME":{"$ref":"#/components/schemas/DateTimeFormatter"},"resolverFields":{"items":{"type":"object"},"type":"array","uniqueItems":true},"ISO_LOCAL_DATE_TIME":{"$ref":"#/components/schemas/DateTimeFormatter"},"PARSED_EXCESS_DAYS":{"$ref":"#/components/schemas/TemporalQuery"},"decimalStyle":{"$ref":"#/components/schemas/DecimalStyle"},"RFC_1123_DATE_TIME":{"$ref":"#/components/schemas/DateTimeFormatter"},"ISO_INSTANT":{"$ref":"#/components/schemas/DateTimeFormatter"},"ISO_ZONED_DATE_TIME":{"$ref":"#/components/schemas/DateTimeFormatter"},"ISO_OFFSET_DATE_TIME":{"$ref":"#/components/schemas/DateTimeFormatter"},"ISO_DATE_TIME":{"$ref":"#/components/schemas/DateTimeFormatter"},"locale":{"$ref":"#/components/schemas/Locale"},"ISO_DATE":{"$ref":"#/components/schemas/DateTimeFormatter"},"ISO_TIME":{"$ref":"#/components/schemas/DateTimeFormatter"},"ISO_LOCAL_TIME":{"$ref":"#/components/schemas/DateTimeFormatter"},"ISO_OFFSET_DATE":{"$ref":"#/components/schemas/DateTimeFormatter"},"zone":{"$ref":"#/components/schemas/ZoneId"},"ISO_ORDINAL_DATE":{"$ref":"#/components/schemas/DateTimeFormatter"},"resolverStyle":{"enums":["STRICT","SMART","LENIENT"],"type":"string"},"ISO_LOCAL_DATE":{"$ref":"#/components/schemas/DateTimeFormatter"},"chrono":{"$ref":"#/components/schemas/Chronology"},"BASIC_ISO_DATE":{"$ref":"#/components/schemas/DateTimeFormatter"},"ISO_WEEK_DATE":{"$ref":"#/components/schemas/DateTimeFormatter"},"PARSED_LEAP_SECOND":{"$ref":"#/components/schemas/TemporalQuery"}},"type":"object"},"Certificate":{"name":"Certificate","properties":{"serialVersionUID":{"format":"int64","type":"integer"},"type":{"type":"string"},"hash":{"format":"int32","type":"integer"}},"type":"object"},"Cache":{"name":"Cache","properties":{"map":{"$ref":"#/components/schemas/ConcurrentMap"},"queue":{"$ref":"#/components/schemas/ReferenceQueue"}},"type":"object"},"ConstructorAccessor":{"name":"ConstructorAccessor","type":"object"},"PermissionCollection":{"name":"PermissionCollection","properties":{"serialVersionUID":{"format":"int64","type":"integer"},"readOnly":{"type":"boolean"}},"type":"object"},"HttpEntity":{"name":"HttpEntity","properties":{"headers":{"$ref":"#/components/schemas/HttpHeaders"},"body":{"$ref":"#/components/schemas/Object"},"EMPTY":{"$ref":"#/components/schemas/HttpEntity"}},"type":"object"},"AnnotationData":{"name":"AnnotationData","properties":{"declaredAnnotations":{"$ref":"#/components/schemas/Map"},"redefinedCount":{"format":"int32","type":"integer"},"annotations":{"$ref":"#/components/schemas/Map"}},"type":"object"},"ModelAndView":{"name":"ModelAndView","properties":{"view":{"$ref":"#/components/schemas/Object"},"model":{"$ref":"#/components/schemas/ModelMap"},"cleared":{"type":"boolean"},"status":{"enums":["100 CONTINUE","101 SWITCHING_PROTOCOLS","102 PROCESSING","103 CHECKPOINT","200 OK","201 CREATED","202 ACCEPTED","203 NON_AUTHORITATIVE_INFORMATION","204 NO_CONTENT","205 RESET_CONTENT","206 PARTIAL_CONTENT","207 MULTI_STATUS","208 ALREADY_REPORTED","226 IM_USED","300 MULTIPLE_CHOICES","301 MOVED_PERMANENTLY","302 FOUND","302 MOVED_TEMPORARILY","303 SEE_OTHER","304 NOT_MODIFIED","305 USE_PROXY","307 TEMPORARY_REDIRECT","308 PERMANENT_REDIRECT","400 BAD_REQUEST","401 UNAUTHORIZED","402 PAYMENT_REQUIRED","403 FORBIDDEN","404 NOT_FOUND","405 METHOD_NOT_ALLOWED","406 NOT_ACCEPTABLE","407 PROXY_AUTHENTICATION_REQUIRED","408 REQUEST_TIMEOUT","409 CONFLICT","410 GONE","411 LENGTH_REQUIRED","412 PRECONDITION_FAILED","413 PAYLOAD_TOO_LARGE","413 REQUEST_ENTITY_TOO_LARGE","414 URI_TOO_LONG","414 REQUEST_URI_TOO_LONG","415 UNSUPPORTED_MEDIA_TYPE","416 REQUESTED_RANGE_NOT_SATISFIABLE","417 EXPECTATION_FAILED","418 I_AM_A_TEAPOT","419 INSUFFICIENT_SPACE_ON_RESOURCE","420 METHOD_FAILURE","421 DESTINATION_LOCKED","422 UNPROCESSABLE_ENTITY","423 LOCKED","424 FAILED_DEPENDENCY","425 TOO_EARLY","426 UPGRADE_REQUIRED","428 PRECONDITION_REQUIRED","429 TOO_MANY_REQUESTS","431 REQUEST_HEADER_FIELDS_TOO_LARGE","451 UNAVAILABLE_FOR_LEGAL_REASONS","500 INTERNAL_SERVER_ERROR","501 NOT_IMPLEMENTED","502 BAD_GATEWAY","503 SERVICE_UNAVAILABLE","504 GATEWAY_TIMEOUT","505 HTTP_VERSION_NOT_SUPPORTED","506 VARIANT_ALSO_NEGOTIATES","507 INSUFFICIENT_STORAGE","508 LOOP_DETECTED","509 BANDWIDTH_LIMIT_EXCEEDED","510 NOT_EXTENDED","511 NETWORK_AUTHENTICATION_REQUIRED"],"type":"string"}},"type":"object"},"URLStreamHandlerFactory":{"name":"URLStreamHandlerFactory","type":"object"},"Properties":{"name":"Properties","properties":{"entrySet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"values":{"$ref":"#/components/schemas/Collection"},"count":{"format":"int32","type":"integer"},"threshold":{"format":"int32","type":"integer"},"modCount":{"format":"int32","type":"integer"},"serialVersionUID":{"format":"int64","type":"integer"},"MAX_ARRAY_SIZE":{"format":"int32","type":"integer"},"defaults":{"$ref":"#/components/schemas/Properties"},"loadFactor":{"format":"float","type":"number"},"hexDigit":{"items":{"type":"string"},"type":"array"},"KEYS":{"format":"int32","type":"integer"},"VALUES":{"format":"int32","type":"integer"},"map":{"$ref":"#/components/schemas/ConcurrentHashMap"},"keySet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"table":{"items":{"$ref":"#/components/schemas/Entry"},"type":"array"},"ENTRIES":{"format":"int32","type":"integer"}},"type":"object"},"InetAddressHolder":{"name":"InetAddressHolder","properties":{"originalHostName":{"type":"string"},"hostName":{"type":"string"},"address":{"format":"int32","type":"integer"},"family":{"format":"int32","type":"integer"}},"type":"object"},"FieldRepository":{"name":"FieldRepository","properties":{"factory":{"$ref":"#/components/schemas/GenericsFactory"},"tree":{"$ref":"#/components/schemas/Tree"},"genericType":{"$ref":"#/components/schemas/Type"}},"type":"object"},"FieldAccessor":{"name":"FieldAccessor","type":"object"},"ClassValueMap":{"name":"ClassValueMap","properties":{"INITIAL_ENTRIES":{"format":"int32","type":"integer"},"cacheArray":{"items":{"$ref":"#/components/schemas/Entry"},"type":"array"},"entrySet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"cacheLoadLimit":{"format":"int32","type":"integer"},"values":{"$ref":"#/components/schemas/Collection"},"DEFAULT_LOAD_FACTOR":{"format":"float","type":"number"},"CACHE_LOAD_LIMIT":{"format":"int32","type":"integer"},"PROBE_LIMIT":{"format":"int32","type":"integer"},"$assertionsDisabled":{"type":"boolean"},"threshold":{"format":"int32","type":"integer"},"DEFAULT_INITIAL_CAPACITY":{"format":"int32","type":"integer"},"modCount":{"format":"int32","type":"integer"},"size":{"format":"int32","type":"integer"},"loadFactor":{"format":"float","type":"number"},"cacheLoad":{"format":"int32","type":"integer"},"MAXIMUM_CAPACITY":{"format":"int32","type":"integer"},"keySet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"table":{"items":{"$ref":"#/components/schemas/Entry"},"type":"array"},"queue":{"$ref":"#/components/schemas/ReferenceQueue"},"NULL_KEY":{"$ref":"#/components/schemas/Object"}},"type":"object"},"Field":{"name":"Field","properties":{"genericInfo":{"$ref":"#/components/schemas/FieldRepository"},"declaredAnnotations":{"$ref":"#/components/schemas/Map"},"printStackPropertiesSet":{"type":"boolean"},"overrideFieldAccessor":{"$ref":"#/components/schemas/FieldAccessor"},"signature":{"type":"string"},"annotations":{"items":{"format":"int32","type":"integer"},"type":"array"},"securityCheckCache":{"$ref":"#/components/schemas/Object"},"printStackWhenAccessFails":{"type":"boolean"},"slot":{"format":"int32","type":"integer"},"type":{"$ref":"#/components/schemas/Class"},"modifiers":{"format":"int32","type":"integer"},"fieldAccessor":{"$ref":"#/components/schemas/FieldAccessor"},"ACCESS_PERMISSION":{"$ref":"#/components/schemas/Permission"},"root":{"$ref":"#/components/schemas/Field"},"name":{"type":"string"},"override":{"type":"boolean"},"reflectionFactory":{"$ref":"#/components/schemas/ReflectionFactory"},"clazz":{"$ref":"#/components/schemas/Class"}},"type":"object"},"LocaleExtensions":{"name":"LocaleExtensions","properties":{"NUMBER_THAI":{"$ref":"#/components/schemas/LocaleExtensions"},"extensionMap":{"$ref":"#/components/schemas/Map"},"CALENDAR_JAPANESE":{"$ref":"#/components/schemas/LocaleExtensions"},"$assertionsDisabled":{"type":"boolean"},"id":{"type":"string"}},"type":"object"},"DateTimePrinterParser":{"name":"DateTimePrinterParser","type":"object"},"GroupHead":{"name":"GroupHead","properties":{"next":{"$ref":"#/components/schemas/Node"},"tail":{"$ref":"#/components/schemas/GroupTail"},"localIndex":{"format":"int32","type":"integer"}},"type":"object"},"ClassLoader":{"name":"ClassLoader","properties":{"parent":{"$ref":"#/components/schemas/ClassLoader"},"parallelLockMap":{"$ref":"#/components/schemas/ConcurrentHashMap"},"unnamedModule":{"$ref":"#/components/schemas/Module"},"packageAssertionStatus":{"$ref":"#/components/schemas/Map"},"classLoaderValueMap":{"$ref":"#/components/schemas/ConcurrentHashMap"},"classes":{"items":{"type":"object"},"type":"array"},"$assertionsDisabled":{"type":"boolean"},"usr_paths":{"items":{"type":"string"},"type":"array"},"defaultDomain":{"$ref":"#/components/schemas/ProtectionDomain"},"packages":{"$ref":"#/components/schemas/ConcurrentHashMap"},"nativeLibraryContext":{"items":{"type":"object"},"type":"array"},"sys_paths":{"items":{"type":"string"},"type":"array"},"systemNativeLibraries":{"items":{"type":"object"},"type":"array"},"classAssertionStatus":{"$ref":"#/components/schemas/Map"},"package2certs":{"$ref":"#/components/schemas/Map"},"nocerts":{"items":{"$ref":"#/components/schemas/Certificate"},"type":"array"},"nativeLibraries":{"items":{"type":"object"},"type":"array"},"name":{"type":"string"},"defaultAssertionStatus":{"type":"boolean"},"assertionLock":{"$ref":"#/components/schemas/Object"},"loadedLibraryNames":{"items":{"type":"object"},"type":"array"},"scl":{"$ref":"#/components/schemas/ClassLoader"}},"type":"object"},"Chronology":{"name":"Chronology","type":"object"},"CodeSigner":{"name":"CodeSigner","properties":{"myhash":{"format":"int32","type":"integer"},"serialVersionUID":{"format":"int64","type":"integer"},"signerCertPath":{"$ref":"#/components/schemas/CertPath"},"timestamp":{"$ref":"#/components/schemas/Timestamp"}},"type":"object"},"Debug":{"name":"Debug","properties":{"args":{"type":"string"},"hexDigits":{"items":{"type":"string"},"type":"array"},"prefix":{"type":"string"}},"type":"object"},"BaseCalendar":{"name":"BaseCalendar","properties":{"GREGORIAN_INSTANCE":{"$ref":"#/components/schemas/Gregorian"},"THURSDAY":{"format":"int32","type":"integer"},"MAY":{"format":"int32","type":"integer"},"DECEMBER":{"format":"int32","type":"integer"},"FRIDAY":{"format":"int32","type":"integer"},"FEBRUARY":{"format":"int32","type":"integer"},"WEDNESDAY":{"format":"int32","type":"integer"},"EPOCH_OFFSET":{"format":"int32","type":"integer"},"DAYS_IN_MONTH":{"items":{"format":"int32","type":"integer"},"type":"array"},"DAY_IN_MILLIS":{"format":"int32","type":"integer"},"SATURDAY":{"format":"int32","type":"integer"},"ACCUMULATED_DAYS_IN_MONTH":{"items":{"format":"int32","type":"integer"},"type":"array"},"initialized":{"type":"boolean"},"APRIL":{"format":"int32","type":"integer"},"JANUARY":{"format":"int32","type":"integer"},"JUNE":{"format":"int32","type":"integer"},"namePairs":{"items":{"type":"string"},"type":"array"},"SUNDAY":{"format":"int32","type":"integer"},"MINUTE_IN_MILLIS":{"format":"int32","type":"integer"},"OCTOBER":{"format":"int32","type":"integer"},"TUESDAY":{"format":"int32","type":"integer"},"FIXED_DATES":{"items":{"format":"int32","type":"integer"},"type":"array"},"$assertionsDisabled":{"type":"boolean"},"SEPTEMBER":{"format":"int32","type":"integer"},"NOVEMBER":{"format":"int32","type":"integer"},"HOUR_IN_MILLIS":{"format":"int32","type":"integer"},"eras":{"items":{"$ref":"#/components/schemas/Era"},"type":"array"},"MONDAY":{"format":"int32","type":"integer"},"ACCUMULATED_DAYS_IN_MONTH_LEAP":{"items":{"format":"int32","type":"integer"},"type":"array"},"SECOND_IN_MILLIS":{"format":"int32","type":"integer"},"BASE_YEAR":{"format":"int32","type":"integer"},"names":{"$ref":"#/components/schemas/ConcurrentMap"},"calendars":{"$ref":"#/components/schemas/ConcurrentMap"},"MARCH":{"format":"int32","type":"integer"},"AUGUST":{"format":"int32","type":"integer"},"JULY":{"format":"int32","type":"integer"},"PACKAGE_NAME":{"type":"string"}},"type":"object"},"ZoneId":{"name":"ZoneId","properties":{"serialVersionUID":{"format":"int64","type":"integer"},"SHORT_IDS":{"$ref":"#/components/schemas/Map"}},"type":"object"},"ReferenceQueue":{"name":"ReferenceQueue","properties":{"head":{"$ref":"#/components/schemas/Reference"},"ENQUEUED":{"$ref":"#/components/schemas/ReferenceQueue"},"queueLength":{"format":"int64","type":"integer"},"NULL":{"$ref":"#/components/schemas/ReferenceQueue"},"lock":{"$ref":"#/components/schemas/Lock"},"$assertionsDisabled":{"type":"boolean"}},"type":"object"},"Gregorian":{"name":"Gregorian","properties":{"GREGORIAN_INSTANCE":{"$ref":"#/components/schemas/Gregorian"},"THURSDAY":{"format":"int32","type":"integer"},"MAY":{"format":"int32","type":"integer"},"DECEMBER":{"format":"int32","type":"integer"},"FRIDAY":{"format":"int32","type":"integer"},"FEBRUARY":{"format":"int32","type":"integer"},"WEDNESDAY":{"format":"int32","type":"integer"},"EPOCH_OFFSET":{"format":"int32","type":"integer"},"DAYS_IN_MONTH":{"items":{"format":"int32","type":"integer"},"type":"array"},"DAY_IN_MILLIS":{"format":"int32","type":"integer"},"SATURDAY":{"format":"int32","type":"integer"},"ACCUMULATED_DAYS_IN_MONTH":{"items":{"format":"int32","type":"integer"},"type":"array"},"initialized":{"type":"boolean"},"APRIL":{"format":"int32","type":"integer"},"JANUARY":{"format":"int32","type":"integer"},"JUNE":{"format":"int32","type":"integer"},"namePairs":{"items":{"type":"string"},"type":"array"},"SUNDAY":{"format":"int32","type":"integer"},"MINUTE_IN_MILLIS":{"format":"int32","type":"integer"},"OCTOBER":{"format":"int32","type":"integer"},"TUESDAY":{"format":"int32","type":"integer"},"FIXED_DATES":{"items":{"format":"int32","type":"integer"},"type":"array"},"$assertionsDisabled":{"type":"boolean"},"SEPTEMBER":{"format":"int32","type":"integer"},"NOVEMBER":{"format":"int32","type":"integer"},"HOUR_IN_MILLIS":{"format":"int32","type":"integer"},"eras":{"items":{"$ref":"#/components/schemas/Era"},"type":"array"},"MONDAY":{"format":"int32","type":"integer"},"ACCUMULATED_DAYS_IN_MONTH_LEAP":{"items":{"format":"int32","type":"integer"},"type":"array"},"SECOND_IN_MILLIS":{"format":"int32","type":"integer"},"BASE_YEAR":{"format":"int32","type":"integer"},"names":{"$ref":"#/components/schemas/ConcurrentMap"},"calendars":{"$ref":"#/components/schemas/ConcurrentMap"},"MARCH":{"format":"int32","type":"integer"},"AUGUST":{"format":"int32","type":"integer"},"JULY":{"format":"int32","type":"integer"},"PACKAGE_NAME":{"type":"string"}},"type":"object"},"Date":{"name":"Date","properties":{"zoneOffset":{"format":"int32","type":"integer"},"cachedFixedDateJan1":{"format":"int64","type":"integer"},"hours":{"format":"int32","type":"integer"},"zoneinfo":{"$ref":"#/components/schemas/TimeZone"},"year":{"format":"int32","type":"integer"},"minutes":{"format":"int32","type":"integer"},"normalized":{"type":"boolean"},"forceStandardTime":{"type":"boolean"},"FIELD_UNDEFINED":{"format":"int32","type":"integer"},"cachedYear":{"format":"int32","type":"integer"},"leapYear":{"type":"boolean"},"locale":{"$ref":"#/components/schemas/Locale"},"fraction":{"format":"int64","type":"integer"},"TIME_UNDEFINED":{"format":"int64","type":"integer"},"cachedFixedDateNextJan1":{"format":"int64","type":"integer"},"seconds":{"format":"int32","type":"integer"},"dayOfWeek":{"format":"int32","type":"integer"},"month":{"format":"int32","type":"integer"},"daylightSaving":{"format":"int32","type":"integer"},"era":{"$ref":"#/components/schemas/Era"},"dayOfMonth":{"format":"int32","type":"integer"},"millis":{"format":"int32","type":"integer"}},"type":"object"},"Provider":{"name":"Provider","properties":{"values":{"$ref":"#/components/schemas/Collection"},"threshold":{"format":"int32","type":"integer"},"legacyStrings":{"$ref":"#/components/schemas/Map"},"modCount":{"format":"int32","type":"integer"},"serialVersionUID":{"format":"int64","type":"integer"},"serviceMap":{"$ref":"#/components/schemas/Map"},"hexDigit":{"items":{"type":"string"},"type":"array"},"entrySetCallCount":{"format":"int32","type":"integer"},"KEYS":{"format":"int32","type":"integer"},"initialized":{"type":"boolean"},"VALUES":{"format":"int32","type":"integer"},"legacyMap":{"$ref":"#/components/schemas/Map"},"servicesChanged":{"type":"boolean"},"map":{"$ref":"#/components/schemas/ConcurrentHashMap"},"keySet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"table":{"items":{"$ref":"#/components/schemas/Entry"},"type":"array"},"ENTRIES":{"format":"int32","type":"integer"},"info":{"type":"string"},"serviceSet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"debug":{"$ref":"#/components/schemas/Debug"},"previousKey":{"$ref":"#/components/schemas/ServiceKey"},"knownEngines":{"$ref":"#/components/schemas/Map"},"entrySet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"count":{"format":"int32","type":"integer"},"ALIAS_PREFIX":{"type":"string"},"version":{"format":"double","type":"number"},"versionStr":{"type":"string"},"ALIAS_LENGTH":{"format":"int32","type":"integer"},"MAX_ARRAY_SIZE":{"format":"int32","type":"integer"},"defaults":{"$ref":"#/components/schemas/Properties"},"loadFactor":{"format":"float","type":"number"},"legacyChanged":{"type":"boolean"},"name":{"type":"string"},"ALIAS_PREFIX_LOWER":{"type":"string"}},"type":"object"},"Response":{"name":"Response","properties":{"r":{"$ref":"#/components/schemas/Object"}},"type":"object"},"ModuleDescriptor":{"name":"ModuleDescriptor","properties":{"mainClass":{"type":"string"},"exports":{"items":{"type":"object"},"type":"array","uniqueItems":true},"automatic":{"type":"boolean"},"$assertionsDisabled":{"type":"boolean"},"modifiers":{"items":{"type":"object"},"type":"array","uniqueItems":true},"packages":{"items":{"type":"object"},"type":"array","uniqueItems":true},"version":{"$ref":"#/components/schemas/Version"},"provides":{"items":{"type":"object"},"type":"array","uniqueItems":true},"name":{"type":"string"},"opens":{"items":{"type":"object"},"type":"array","uniqueItems":true},"uses":{"items":{"type":"object"},"type":"array","uniqueItems":true},"rawVersionString":{"type":"string"},"open":{"type":"boolean"},"hash":{"format":"int32","type":"integer"},"requires":{"items":{"type":"object"},"type":"array","uniqueItems":true}},"type":"object"},"Bar":{"name":"Bar","properties":{"l1":{"format":"int64","type":"integer"},"l2":{"items":{"format":"int64","type":"integer"},"type":"array"},"foo":{"$ref":"#/components/schemas/Foo"},"f1":{"format":"float","type":"number"},"f2":{"items":{"format":"float","type":"number"},"type":"array"},"d1":{"format":"double","type":"number"},"d2":{"items":{"format":"double","type":"number"},"type":"array"},"b1":{"type":"boolean"},"b2":{"items":{"type":"boolean"},"type":"array"},"ss1":{"items":{"type":"object"},"type":"array","uniqueItems":true},"bar2":{"items":{"$ref":"#/components/schemas/Bar"},"type":"array"},"bar":{"$ref":"#/components/schemas/Bar"},"foo2":{"items":{"$ref":"#/components/schemas/Foo"},"type":"array"},"s1":{"format":"int32","type":"integer"},"s2":{"items":{"format":"int32","type":"integer"},"type":"array"},"m1":{"$ref":"#/components/schemas/Map"},"k1":{"type":"string"},"k2":{"items":{"type":"string"},"type":"array"},"k3":{"items":{"type":"object"},"type":"array"},"i1":{"format":"int32","type":"integer"},"i2":{"items":{"format":"int32","type":"integer"},"type":"array"},"enumnumnum":{"enums":["A","B","C"],"type":"string"},"c1":{"type":"string"},"c2":{"items":{"type":"string"},"type":"array"},"list1":{"items":{"type":"object"},"type":"array"},"list4":{"items":{"type":"object"},"type":"array"},"list3":{"items":{"type":"object"},"type":"array"},"list2":{"items":{"type":"object"},"type":"array"}},"type":"object"},"ClassLoaderValue":{"name":"ClassLoaderValue","properties":{"JLA":{"$ref":"#/components/schemas/JavaLangAccess"}},"type":"object"}}},"info":{"title":"OpenAPI definition"},"openapi":"3.0.1","paths":{"/request-mapping/multi-path/002":{"delete":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"head":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"options":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"patch":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"put":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"trace":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/error":{"delete":{"operationId":"org.springframework.boot.autoconfigure.web.servlet.error.BasicErrorController#error","responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/ResponseEntity"}}},"description":"ok"}}},"get":{"operationId":"org.springframework.boot.autoconfigure.web.servlet.error.BasicErrorController#error","responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/ResponseEntity"}}},"description":"ok"}}},"head":{"operationId":"org.springframework.boot.autoconfigure.web.servlet.error.BasicErrorController#error","responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/ResponseEntity"}}},"description":"ok"}}},"options":{"operationId":"org.springframework.boot.autoconfigure.web.servlet.error.BasicErrorController#error","responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/ResponseEntity"}}},"description":"ok"}}},"patch":{"operationId":"org.springframework.boot.autoconfigure.web.servlet.error.BasicErrorController#error","responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/ResponseEntity"}}},"description":"ok"}}},"post":{"operationId":"org.springframework.boot.autoconfigure.web.servlet.error.BasicErrorController#error","responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/ResponseEntity"}}},"description":"ok"}}},"put":{"operationId":"org.springframework.boot.autoconfigure.web.servlet.error.BasicErrorController#error","responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/ResponseEntity"}}},"description":"ok"}}},"trace":{"operationId":"org.springframework.boot.autoconfigure.web.servlet.error.BasicErrorController#error","responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/ResponseEntity"}}},"description":"ok"}}}},"/request-mapping/multi-path/001":{"delete":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"head":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"options":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"patch":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"put":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"trace":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/request-mapping/path/{value1}/{value2}":{"delete":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value2","required":false,"schema":{"type":"string"}},{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value2","required":false,"schema":{"type":"string"}},{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"head":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value2","required":false,"schema":{"type":"string"}},{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"options":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value2","required":false,"schema":{"type":"string"}},{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"patch":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value2","required":false,"schema":{"type":"string"}},{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value2","required":false,"schema":{"type":"string"}},{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"put":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value2","required":false,"schema":{"type":"string"}},{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"trace":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value2","required":false,"schema":{"type":"string"}},{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/swagger-ui.html":{"get":{"operationId":"org.springdoc.webmvc.ui.SwaggerWelcomeWebMvc#redirectToUi","responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/ResponseEntity"}}},"description":"ok"}}}},"/delete-mapping":{"delete":{"operationId":"app.iast.api.springmvc.controller.GreetingController#deleteMapping","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/show-parameters/{f1}":{"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#showParameters","parameters":[{"in":"path","name":"f1","required":true,"schema":{"type":"string"}},{"in":"header","name":"f3","required":true,"schema":{"type":"string"}},{"in":"cookie","name":"f4","required":true,"schema":{"type":"string"}},{"in":"query","name":"f5","required":true,"schema":{"type":"string"}}],"requestBody":{"content":{"application/json":{"schema":{"type":"string"}}},"required":true},"responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/Response"}}},"description":"ok"}}}},"/patch-mapping":{"patch":{"operationId":"app.iast.api.springmvc.controller.GreetingController#patch_mapping","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/request-mapping/RequestBody":{"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingHeader0012","parameters":[{"in":"query","name":"name","required":true,"schema":{"type":"string"}}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Foo"}}},"required":true},"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/v3/api-docs":{"get":{"operationId":"org.springdoc.webmvc.api.OpenApiWebMvcResource#openapiJson","parameters":[{"in":"query","name":"apiDocsUrl","required":true,"schema":{"type":"string"}},{"in":"query","name":"locale","required":true,"schema":{"$ref":"#/components/schemas/Locale"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/v3/api-docs/swagger-config":{"get":{"operationId":"org.springdoc.webmvc.ui.SwaggerConfigResource#openapiJson","responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/Map"}}},"description":"ok"}}}},"/request-mapping/produces":{"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingProducesJson","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/request-mapping/RequestParam":{"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingHeader001","parameters":[{"in":"header","name":"foo","required":true,"schema":{"type":"string"}},{"in":"query","name":"name","required":true,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/request-mapping/params":{"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingParamsA1","parameters":[{"in":"query","name":"action","required":true,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/get-mapping":{"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#getMapping","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/generic":{"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#generic","parameters":[{"in":"query","name":"foo","required":true,"schema":{"$ref":"#/components/schemas/Foo"}},{"in":"query","name":"name","required":true,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/Response"}}},"description":"ok"}}}},"/*":{"delete":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingFallback","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingFallback","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"head":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingFallback","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"options":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingFallback","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"patch":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingFallback","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingFallback","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"put":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingFallback","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"trace":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingFallback","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/request-mapping/multi-method":{"delete":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiMethod","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiMethod","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiMethod","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"put":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiMethod","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/test-openapi-data-types":{"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#testOpenApiDataTypes","parameters":[{"in":"query","name":"v6","required":false,"schema":{"type":"boolean"}},{"in":"query","name":"v7","required":false,"schema":{"type":"string"}},{"in":"query","name":"v8","required":false,"schema":{"format":"int32","type":"integer"}},{"in":"query","name":"v9","required":false,"schema":{"items":{"format":"int32","type":"integer"},"type":"array"}},{"in":"query","name":"v10","required":false,"schema":{"$ref":"#/components/schemas/Foo"}},{"in":"query","name":"v1","required":false,"schema":{"type":"string"}},{"in":"query","name":"v2","required":false,"schema":{"format":"float","type":"number"}},{"in":"query","name":"v3","required":false,"schema":{"format":"double","type":"number"}},{"in":"query","name":"v4","required":false,"schema":{"format":"int32","type":"integer"}},{"in":"query","name":"v5","required":false,"schema":{"format":"int64","type":"integer"}}],"responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/Foo"}}},"description":"ok"}}}},"/post-mapping":{"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#postMapping","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/request-mapping/headers":{"delete":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingHeader001","parameters":[{"in":"header","name":"foo","required":true,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingHeader001","parameters":[{"in":"header","name":"foo","required":true,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"head":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingHeader001","parameters":[{"in":"header","name":"foo","required":true,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"options":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingHeader001","parameters":[{"in":"header","name":"foo","required":true,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"patch":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingHeader001","parameters":[{"in":"header","name":"foo","required":true,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingHeader001","parameters":[{"in":"header","name":"foo","required":true,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"put":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingHeader001","parameters":[{"in":"header","name":"foo","required":true,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"trace":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingHeader001","parameters":[{"in":"header","name":"foo","required":true,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/request-mapping/consumes":{"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingConsumersXml","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/request-mapping/path/{numericId:\\d+}":{"delete":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariableRegexp","parameters":[{"in":"path","name":"numericId","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariableRegexp","parameters":[{"in":"path","name":"numericId","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"head":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariableRegexp","parameters":[{"in":"path","name":"numericId","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"options":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariableRegexp","parameters":[{"in":"path","name":"numericId","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"patch":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariableRegexp","parameters":[{"in":"path","name":"numericId","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariableRegexp","parameters":[{"in":"path","name":"numericId","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"put":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariableRegexp","parameters":[{"in":"path","name":"numericId","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"trace":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariableRegexp","parameters":[{"in":"path","name":"numericId","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/v3/api-docs.yaml":{"get":{"operationId":"org.springdoc.webmvc.api.OpenApiWebMvcResource#openapiYaml","parameters":[{"in":"query","name":"apiDocsUrl","required":true,"schema":{"type":"string"}},{"in":"query","name":"locale","required":true,"schema":{"$ref":"#/components/schemas/Locale"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/request-mapping/path/{value1}":{"delete":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"head":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"options":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"patch":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"put":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"trace":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/test-spring-mvc-api-collect":{"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#testSpringMVCApiCollect","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/put-mapping":{"put":{"operationId":"app.iast.api.springmvc.controller.GreetingController#putMapping","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/request-mapping":{"delete":{"operationId":"app.iast.api.springmvc.controller.GreetingController#greeting","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#greeting","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"head":{"operationId":"app.iast.api.springmvc.controller.GreetingController#greeting","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"options":{"operationId":"app.iast.api.springmvc.controller.GreetingController#greeting","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"patch":{"operationId":"app.iast.api.springmvc.controller.GreetingController#greeting","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#greeting","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"put":{"operationId":"app.iast.api.springmvc.controller.GreetingController#greeting","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"trace":{"operationId":"app.iast.api.springmvc.controller.GreetingController#greeting","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}}}}}} \ No newline at end of file From c06e1b2c7198cf4cde9f08093d5ac85dbb823992 Mon Sep 17 00:00:00 2001 From: bidaya0 Date: Wed, 21 Jun 2023 16:31:25 +0800 Subject: [PATCH 008/161] feat/new_api_route_upload --- dongtai_protocol/report/handler/api_route_v2_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dongtai_protocol/report/handler/api_route_v2_handler.py b/dongtai_protocol/report/handler/api_route_v2_handler.py index 27a4a5675..00c573a17 100644 --- a/dongtai_protocol/report/handler/api_route_v2_handler.py +++ b/dongtai_protocol/report/handler/api_route_v2_handler.py @@ -51,7 +51,7 @@ def get_schama_key(ref_str: str) -> str: def validate_schema_key(test_str: str) -> bool: - return re.fullmatch(regex, test_str) is not None + return re.fullmatch(r"#\/components\/schemas\/.*", test_str) is not None @shared_task(queue='dongtai-api-route-handler') From e7a63b35cae6d4d8f68793ea2f1eec2c84d7792e Mon Sep 17 00:00:00 2001 From: Bidaya0 Date: Sun, 25 Jun 2023 12:08:54 +0800 Subject: [PATCH 009/161] Revert "feat/new_api_route_upload" --- dongtai_common/models/api_route_v2.py | 40 ------- dongtai_common/utils/const.py | 1 - dongtai_conf/celery.py | 1 - dongtai_protocol/report/__init__.py | 5 +- .../report/handler/api_route_v2_handler.py | 104 ------------------ .../handler/saas_method_pool_handler.py | 2 +- test/apiserver/test_agent_apiroute_v2.py | 16 --- test/integration/mockdata/api-report.json | 1 - 8 files changed, 2 insertions(+), 168 deletions(-) delete mode 100644 dongtai_common/models/api_route_v2.py delete mode 100644 dongtai_protocol/report/handler/api_route_v2_handler.py delete mode 100644 test/apiserver/test_agent_apiroute_v2.py delete mode 100644 test/integration/mockdata/api-report.json diff --git a/dongtai_common/models/api_route_v2.py b/dongtai_common/models/api_route_v2.py deleted file mode 100644 index 55a9931db..000000000 --- a/dongtai_common/models/api_route_v2.py +++ /dev/null @@ -1,40 +0,0 @@ -###################################################################### -# @author : bidaya0 (bidaya0@$HOSTNAME) -# @file : api_route -# @created : Tuesday Aug 17, 2021 17:43:27 CST -# -# @description : -###################################################################### - -from django.db import models -from dongtai_common.utils.settings import get_managed -from dongtai_common.models.agent import IastAgent -from dongtai_common.models.project import IastProject -from dongtai_common.models.project_version import IastProjectVersion - - -class FromWhereChoices(models.IntegerChoices): - FROM_AGENT = 1 - - -class IastApiRouteV2(models.Model): - path = models.CharField(max_length=255, blank=True) - method = models.CharField(max_length=100, blank=True) - from_where = models.IntegerField(default=FromWhereChoices.FROM_AGENT, - choices=FromWhereChoices.choices) - project = models.ForeignKey(IastProject, - on_delete=models.CASCADE, - blank=True, - null=True, - default=-1) - project_version = models.ForeignKey(IastProjectVersion, - on_delete=models.CASCADE, - blank=True, - null=True, - default=-1) - is_cover = models.IntegerField(default=0) - api_info = models.JSONField(blank=True, null=True, default=dict) - - class Meta: - managed = get_managed() - db_table = 'iast_api_route_v2' diff --git a/dongtai_common/utils/const.py b/dongtai_common/utils/const.py index 5eb6d65aa..d4ac70f27 100644 --- a/dongtai_common/utils/const.py +++ b/dongtai_common/utils/const.py @@ -16,7 +16,6 @@ REPORT_AUTH_UPDATE = 0x32 REPORT_ERROR_LOG = 0x51 REPORT_API_ROUTE = 0x61 -REPORT_API_ROUTE_V2 = 0x62 REPORT_THIRD_PARTY_SERVICE = 0x81 REPORT_FILE_PATH = 0x82 diff --git a/dongtai_conf/celery.py b/dongtai_conf/celery.py index 4be740b19..1c82272b2 100644 --- a/dongtai_conf/celery.py +++ b/dongtai_conf/celery.py @@ -68,7 +68,6 @@ # configs['worker_concurrency'] = 8 configs["task_routes"] = { # normal - "dongtai_protocol.report.handler.api_route_v2_handler.api_route_gather_v2": {'queue': 'dongtai-api-route-handler', 'routing_key': 'dongtai-api-route-handler'}, "dongtai_protocol.report.handler.api_route_handler.api_route_gather": {'queue': 'dongtai-api-route-handler', 'routing_key': 'dongtai-api-route-handler'}, "dongtai_engine.tasks.search_vul_from_method_pool": {'queue': 'dongtai-method-pool-scan', 'routing_key': 'dongtai-method-pool-scan'}, "dongtai_engine.plugins.project_time_update.project_time_stamp_update": {'queue': 'dongtai-project-time-stamp-update', 'routing_key': 'dongtai-project-time-stamp-update'}, diff --git a/dongtai_protocol/report/__init__.py b/dongtai_protocol/report/__init__.py index 1aac0b5fa..18cd8cf2e 100644 --- a/dongtai_protocol/report/__init__.py +++ b/dongtai_protocol/report/__init__.py @@ -8,10 +8,8 @@ from dongtai_protocol.report.handler.heartbeat_handler import HeartBeatHandler from dongtai_protocol.report.handler.narmal_vul_handler import NormalVulnHandler from dongtai_protocol.report.handler.saas_method_pool_handler import SaasMethodPoolHandler -from dongtai_protocol.report.handler.sca_handler import (ScaHandler, - ScaBulkHandler) +from dongtai_protocol.report.handler.sca_handler import (ScaHandler, ScaBulkHandler) from dongtai_protocol.report.handler.api_route_handler import ApiRouteHandler -from dongtai_protocol.report.handler.api_route_v2_handler import ApiRouteV2Handler from dongtai_protocol.report.handler.hardencode_vul_handler import HardEncodeVulHandler from dongtai_protocol.report.handler.agent_third_service_handler import ThirdPartyServiceHandler from dongtai_protocol.report.handler.agent_filepath_handler import FilePathHandler @@ -23,7 +21,6 @@ NormalVulnHandler() SaasMethodPoolHandler() ApiRouteHandler() - ApiRouteV2Handler() HardEncodeVulHandler() ScaBulkHandler() ThirdPartyServiceHandler() diff --git a/dongtai_protocol/report/handler/api_route_v2_handler.py b/dongtai_protocol/report/handler/api_route_v2_handler.py deleted file mode 100644 index 00c573a17..000000000 --- a/dongtai_protocol/report/handler/api_route_v2_handler.py +++ /dev/null @@ -1,104 +0,0 @@ -###################################################################### -# @author : bidaya0 (bidaya0@$HOSTNAME) -# @file : api_route_handler -# @created : Tuesday Aug 17, 2021 19:59:29 CST -# -# @description : -###################################################################### - -from dongtai_protocol.report.handler.report_handler_interface import IReportHandler -from dongtai_protocol.report.report_handler_factory import ReportHandler -from dongtai_common.models.api_route_v2 import IastApiRouteV2, FromWhereChoices -from dongtai_common.models.agent import IastAgent -from dongtai_common.utils import const -import logging -from django.utils.translation import gettext_lazy as _ -from django.db import transaction -from dongtai_common.models.project import IastProject -from dongtai_engine.plugins.project_time_update import project_time_stamp_update -from celery import shared_task -import re - -logger = logging.getLogger('dongtai.openapi') - - -@ReportHandler.register(const.REPORT_API_ROUTE_V2) -class ApiRouteV2Handler(IReportHandler): - - def parse(self): - self.api_data = self.detail.get('apiData') - - def save(self): - api_route_gather_v2.delay(self.agent_id, self.api_data) - - -def find_values_with_key(dictionary, key): - values = [] - for dic_key, value in dictionary.items(): - if isinstance(value, dict): - values.extend(find_values_with_key(value, key)) - if isinstance(value, list): - for item in value: - if isinstance(item, dict): - values.extend(find_values_with_key(item, key)) - if key == dic_key: - values.append(value) - return values - - -def get_schama_key(ref_str: str) -> str: - return ref_str.replace("#/components/schemas/", "") - - -def validate_schema_key(test_str: str) -> bool: - return re.fullmatch(r"#\/components\/schemas\/.*", test_str) is not None - - -@shared_task(queue='dongtai-api-route-handler') -def api_route_gather_v2(agent_id, api_routes): - logger.info(_('API navigation log start '), ) - try: - agent = IastAgent.objects.get(pk=agent_id) - - schemas = api_routes['components']['schemas'] - - api_routes_list = [] - for path, path_info in api_routes['paths'].items(): - for method, api in path_info.items(): - ref_in_api = find_values_with_key(api, '$ref') - schema_keys = [get_schama_key(i) for i in ref_in_api] - extenal_schema_keys = [] - for key in schema_keys: - extenal_schema_keys.extend( - find_values_with_key(schemas[key], "$ref")) - extenal_schema_keys = [ - get_schama_key(i) for i in extenal_schema_keys - ] - keys = set(schema_keys + extenal_schema_keys) - new_dict = { - "paths": { - path: { - method: api - } - }, - "components": { - "schemas": {key: schemas[key] - for key in keys} - } - } - api_route = IastApiRouteV2( - path=path, - method=method, - from_where=FromWhereChoices.FROM_AGENT, - project_id=agent.bind_project_id, - project_version_id=agent.project_version_id, - api_info=new_dict, - ) - api_routes_list.append(api_route) - IastApiRouteV2.objects.bulk_create(api_routes_list, - ignore_conflicts=False) - except Exception as e: - logger.info( - _('API navigation log failed, why: {}').format(e), - exc_info=e, - ) diff --git a/dongtai_protocol/report/handler/saas_method_pool_handler.py b/dongtai_protocol/report/handler/saas_method_pool_handler.py index f9e77da02..ff7fa19ab 100644 --- a/dongtai_protocol/report/handler/saas_method_pool_handler.py +++ b/dongtai_protocol/report/handler/saas_method_pool_handler.py @@ -105,7 +105,7 @@ def save(self): """ headers = SaasMethodPoolHandler.parse_headers(self.http_req_header) #save_project_header(list(headers.keys()), self.agent_id) - #add_new_api_route(self.agent, self.http_uri, self.http_method) + add_new_api_route(self.agent, self.http_uri, self.http_method) import base64 params_dict = get_params_dict(base64.b64decode(self.http_req_header), self.http_req_data, diff --git a/test/apiserver/test_agent_apiroute_v2.py b/test/apiserver/test_agent_apiroute_v2.py deleted file mode 100644 index 2b8b7a55c..000000000 --- a/test/apiserver/test_agent_apiroute_v2.py +++ /dev/null @@ -1,16 +0,0 @@ -from test.apiserver.test_agent_base import AgentTestCase -from dongtai_common.models.api_route_v2 import IastApiRouteV2 -from dongtai_common.models.agent import IastAgent -import json - - -class ApiRouteV2TestCase(AgentTestCase): - - def test_agent_api_upload(self): - with open('./test/integration/mockdata/api-report.json') as fp: - data = json.load(fp) - data['detail']['agentId'] = self.agent_id - res = self.agent_report(data, agentId=self.agent_id) - self.assertGreater( - IastApiRouteV2.objects.filter( - project_version__iastagent__pk=self.agent_id).count(), 0) diff --git a/test/integration/mockdata/api-report.json b/test/integration/mockdata/api-report.json deleted file mode 100644 index d1cccee5f..000000000 --- a/test/integration/mockdata/api-report.json +++ /dev/null @@ -1 +0,0 @@ -{"type":98,"detail":{"agentId":11,"framework":"spring mvc","apiData":{"components":{"schemas":{"HttpHeaders":{"name":"HttpHeaders","properties":{"CONTENT_LANGUAGE":{"type":"string"},"HOST":{"type":"string"},"IF_MODIFIED_SINCE":{"type":"string"},"FROM":{"type":"string"},"CACHE_CONTROL":{"type":"string"},"CONTENT_RANGE":{"type":"string"},"IF_RANGE":{"type":"string"},"CONTENT_TYPE":{"type":"string"},"PRAGMA":{"type":"string"},"VARY":{"type":"string"},"ALLOW":{"type":"string"},"IF_MATCH":{"type":"string"},"ACCEPT_ENCODING":{"type":"string"},"CONTENT_LOCATION":{"type":"string"},"TRANSFER_ENCODING":{"type":"string"},"USER_AGENT":{"type":"string"},"ACCEPT_RANGES":{"type":"string"},"ACCESS_CONTROL_ALLOW_HEADERS":{"type":"string"},"DATE_FORMATTER":{"$ref":"#/components/schemas/DateTimeFormatter"},"ACCESS_CONTROL_ALLOW_ORIGIN":{"type":"string"},"WARNING":{"type":"string"},"IF_NONE_MATCH":{"type":"string"},"GMT":{"$ref":"#/components/schemas/ZoneId"},"DATE":{"type":"string"},"ACCEPT":{"type":"string"},"CONTENT_LENGTH":{"type":"string"},"ETAG":{"type":"string"},"COOKIE":{"type":"string"},"RETRY_AFTER":{"type":"string"},"TRAILER":{"type":"string"},"ETAG_HEADER_VALUE_PATTERN":{"$ref":"#/components/schemas/Pattern"},"RANGE":{"type":"string"},"LOCATION":{"type":"string"},"CONTENT_DISPOSITION":{"type":"string"},"IF_UNMODIFIED_SINCE":{"type":"string"},"LAST_MODIFIED":{"type":"string"},"WWW_AUTHENTICATE":{"type":"string"},"ACCESS_CONTROL_ALLOW_CREDENTIALS":{"type":"string"},"serialVersionUID":{"format":"int64","type":"integer"},"ORIGIN":{"type":"string"},"SET_COOKIE2":{"type":"string"},"ACCESS_CONTROL_MAX_AGE":{"type":"string"},"EXPIRES":{"type":"string"},"SERVER":{"type":"string"},"LINK":{"type":"string"},"EMPTY":{"$ref":"#/components/schemas/HttpHeaders"},"CONTENT_ENCODING":{"type":"string"},"AGE":{"type":"string"},"DATE_PARSERS":{"items":{"$ref":"#/components/schemas/DateTimeFormatter"},"type":"array"},"headers":{"$ref":"#/components/schemas/MultiValueMap"},"EXPECT":{"type":"string"},"ACCESS_CONTROL_REQUEST_METHOD":{"type":"string"},"SET_COOKIE":{"type":"string"},"PROXY_AUTHORIZATION":{"type":"string"},"VIA":{"type":"string"},"ACCESS_CONTROL_REQUEST_HEADERS":{"type":"string"},"CONNECTION":{"type":"string"},"MAX_FORWARDS":{"type":"string"},"TE":{"type":"string"},"ACCEPT_LANGUAGE":{"type":"string"},"REFERER":{"type":"string"},"ACCEPT_CHARSET":{"type":"string"},"ACCESS_CONTROL_ALLOW_METHODS":{"type":"string"},"UPGRADE":{"type":"string"},"PROXY_AUTHENTICATE":{"type":"string"},"ACCESS_CONTROL_EXPOSE_HEADERS":{"type":"string"},"DECIMAL_FORMAT_SYMBOLS":{"$ref":"#/components/schemas/DecimalFormatSymbols"},"AUTHORIZATION":{"type":"string"}},"type":"object"},"CertificateFactorySpi":{"name":"CertificateFactorySpi","type":"object"},"Configuration":{"name":"Configuration","properties":{"EMPTY_CONFIGURATION":{"$ref":"#/components/schemas/Configuration"},"allConfigurations":{"items":{"type":"object"},"type":"array"},"nameToModule":{"$ref":"#/components/schemas/Map"},"targetPlatform":{"type":"string"},"graph":{"$ref":"#/components/schemas/Map"},"modules":{"items":{"type":"object"},"type":"array","uniqueItems":true},"parents":{"items":{"type":"object"},"type":"array"}},"type":"object"},"Node":{"name":"Node","properties":{"next":{"$ref":"#/components/schemas/Node"},"value":{"$ref":"#/components/schemas/Object"},"hash":{"format":"int32","type":"integer"},"key":{"$ref":"#/components/schemas/Object"}},"type":"object"},"CharPredicate":{"name":"CharPredicate","type":"object"},"DecimalStyle":{"name":"DecimalStyle","properties":{"negativeSign":{"type":"string"},"decimalSeparator":{"type":"string"},"zeroDigit":{"type":"string"},"CACHE":{"$ref":"#/components/schemas/ConcurrentMap"},"positiveSign":{"type":"string"},"STANDARD":{"$ref":"#/components/schemas/DecimalStyle"}},"type":"object"},"SoftReference":{"name":"SoftReference","properties":{"next":{"$ref":"#/components/schemas/Reference"},"discovered":{"$ref":"#/components/schemas/Reference"},"referent":{"$ref":"#/components/schemas/Object"},"processPendingActive":{"type":"boolean"},"clock":{"format":"int64","type":"integer"},"queue":{"$ref":"#/components/schemas/ReferenceQueue"},"timestamp":{"format":"int64","type":"integer"},"processPendingLock":{"$ref":"#/components/schemas/Object"}},"type":"object"},"ThreadLocal":{"name":"ThreadLocal","properties":{"HASH_INCREMENT":{"format":"int32","type":"integer"},"threadLocalHashCode":{"format":"int32","type":"integer"},"nextHashCode":{"$ref":"#/components/schemas/AtomicInteger"}},"type":"object"},"TypeVariable":{"name":"TypeVariable","type":"object"},"Currency":{"name":"Currency","properties":{"instances":{"$ref":"#/components/schemas/ConcurrentMap"},"NUMERIC_CODE_SHIFT":{"format":"int32","type":"integer"},"available":{"items":{"type":"object"},"type":"array","uniqueItems":true},"VALID_FORMAT_VERSION":{"format":"int32","type":"integer"},"otherCurrenciesList":{"items":{"type":"object"},"type":"array"},"INVALID_COUNTRY_ENTRY":{"format":"int32","type":"integer"},"SIMPLE_CASE_COUNTRY_MASK":{"format":"int32","type":"integer"},"SIMPLE_CASE_COUNTRY_MAX_DEFAULT_DIGITS":{"format":"int32","type":"integer"},"numericCode":{"format":"int32","type":"integer"},"serialVersionUID":{"format":"int64","type":"integer"},"COUNTRY_TYPE_MASK":{"format":"int32","type":"integer"},"SIMPLE_CASE_COUNTRY_DEFAULT_DIGITS_SHIFT":{"format":"int32","type":"integer"},"MAGIC_NUMBER":{"format":"int32","type":"integer"},"A_TO_Z":{"format":"int32","type":"integer"},"formatVersion":{"format":"int32","type":"integer"},"COUNTRY_WITHOUT_CURRENCY_ENTRY":{"format":"int32","type":"integer"},"DISPLAYNAME":{"format":"int32","type":"integer"},"specialCasesList":{"items":{"type":"object"},"type":"array"},"dataVersion":{"format":"int32","type":"integer"},"SIMPLE_CASE_COUNTRY_FINAL_CHAR_MASK":{"format":"int32","type":"integer"},"defaultFractionDigits":{"format":"int32","type":"integer"},"mainTable":{"items":{"format":"int32","type":"integer"},"type":"array"},"SPECIAL_CASE_COUNTRY_INDEX_DELTA":{"format":"int32","type":"integer"},"NUMERIC_CODE_MASK":{"format":"int32","type":"integer"},"SYMBOL":{"format":"int32","type":"integer"},"SPECIAL_CASE_COUNTRY_MASK":{"format":"int32","type":"integer"},"currencyCode":{"type":"string"},"SIMPLE_CASE_COUNTRY_DEFAULT_DIGITS_MASK":{"format":"int32","type":"integer"},"SPECIAL_CASE_COUNTRY_INDEX_MASK":{"format":"int32","type":"integer"}},"type":"object"},"Permission":{"name":"Permission","properties":{"serialVersionUID":{"format":"int64","type":"integer"},"name":{"type":"string"}},"type":"object"},"TemporalQuery":{"name":"TemporalQuery","type":"object"},"JavaLangAccess":{"name":"JavaLangAccess","type":"object"},"MethodAccessor":{"name":"MethodAccessor","type":"object"},"ServicesCatalog":{"name":"ServicesCatalog","properties":{"CLV":{"$ref":"#/components/schemas/ClassLoaderValue"},"map":{"$ref":"#/components/schemas/Map"}},"type":"object"},"Method":{"name":"Method","properties":{"genericInfo":{"$ref":"#/components/schemas/MethodRepository"},"hasRealParameterData":{"type":"boolean"},"declaredAnnotations":{"$ref":"#/components/schemas/Map"},"printStackPropertiesSet":{"type":"boolean"},"parameterTypes":{"items":{"$ref":"#/components/schemas/Class"},"type":"array"},"signature":{"type":"string"},"annotations":{"items":{"format":"int32","type":"integer"},"type":"array"},"securityCheckCache":{"$ref":"#/components/schemas/Object"},"printStackWhenAccessFails":{"type":"boolean"},"slot":{"format":"int32","type":"integer"},"modifiers":{"format":"int32","type":"integer"},"methodAccessor":{"$ref":"#/components/schemas/MethodAccessor"},"ACCESS_PERMISSION":{"$ref":"#/components/schemas/Permission"},"exceptionTypes":{"items":{"$ref":"#/components/schemas/Class"},"type":"array"},"annotationDefault":{"items":{"format":"int32","type":"integer"},"type":"array"},"root":{"$ref":"#/components/schemas/Method"},"name":{"type":"string"},"parameterAnnotations":{"items":{"format":"int32","type":"integer"},"type":"array"},"override":{"type":"boolean"},"reflectionFactory":{"$ref":"#/components/schemas/ReflectionFactory"},"clazz":{"$ref":"#/components/schemas/Class"},"parameters":{"items":{"$ref":"#/components/schemas/Parameter"},"type":"array"},"returnType":{"$ref":"#/components/schemas/Class"}},"type":"object"},"CodeSource":{"name":"CodeSource","properties":{"signers":{"items":{"$ref":"#/components/schemas/CodeSigner"},"type":"array"},"factory":{"$ref":"#/components/schemas/CertificateFactory"},"serialVersionUID":{"format":"int64","type":"integer"},"location":{"$ref":"#/components/schemas/URL"},"locationNoFragString":{"type":"string"},"certs":{"items":{"$ref":"#/components/schemas/Certificate"},"type":"array"},"sp":{"$ref":"#/components/schemas/SocketPermission"}},"type":"object"},"URLStreamHandler":{"name":"URLStreamHandler","type":"object"},"TimeZone":{"name":"TimeZone","properties":{"ONE_HOUR":{"format":"int32","type":"integer"},"GMT_ID":{"type":"string"},"$assertionsDisabled":{"type":"boolean"},"GMT_ID_LENGTH":{"format":"int32","type":"integer"},"SHORT":{"format":"int32","type":"integer"},"serialVersionUID":{"format":"int64","type":"integer"},"defaultTimeZone":{"$ref":"#/components/schemas/TimeZone"},"mainAppContextDefault":{"$ref":"#/components/schemas/TimeZone"},"NO_TIMEZONE":{"$ref":"#/components/schemas/TimeZone"},"zoneId":{"$ref":"#/components/schemas/ZoneId"},"ID":{"type":"string"},"ONE_DAY":{"format":"int32","type":"integer"},"ONE_MINUTE":{"format":"int32","type":"integer"},"LONG":{"format":"int32","type":"integer"}},"type":"object"},"AtomicInteger":{"name":"AtomicInteger","properties":{"serialVersionUID":{"format":"int64","type":"integer"},"U":{"$ref":"#/components/schemas/Unsafe"},"VALUE":{"format":"int64","type":"integer"},"value":{"format":"int32","type":"integer"}},"type":"object"},"Era":{"name":"Era","properties":{"localTime":{"type":"boolean"},"name":{"type":"string"},"abbr":{"type":"string"},"sinceDate":{"$ref":"#/components/schemas/CalendarDate"},"hash":{"format":"int32","type":"integer"},"since":{"format":"int64","type":"integer"}},"type":"object"},"Object":{"name":"Object","type":"object"},"ModuleLayer":{"name":"ModuleLayer","properties":{"cf":{"$ref":"#/components/schemas/Configuration"},"servicesCatalog":{"$ref":"#/components/schemas/ServicesCatalog"},"EMPTY_LAYER":{"$ref":"#/components/schemas/ModuleLayer"},"nameToModule":{"$ref":"#/components/schemas/Map"},"CLV":{"$ref":"#/components/schemas/ClassLoaderValue"},"allLayers":{"items":{"type":"object"},"type":"array"},"modules":{"items":{"type":"object"},"type":"array","uniqueItems":true},"parents":{"items":{"type":"object"},"type":"array"}},"type":"object"},"Module":{"name":"Module","properties":{"EVERYONE_MODULE":{"$ref":"#/components/schemas/Module"},"loader":{"$ref":"#/components/schemas/ClassLoader"},"openPackages":{"$ref":"#/components/schemas/Map"},"moduleInfoClass":{"$ref":"#/components/schemas/Class"},"reads":{"items":{"type":"object"},"type":"array","uniqueItems":true},"$assertionsDisabled":{"type":"boolean"},"descriptor":{"$ref":"#/components/schemas/ModuleDescriptor"},"exportedPackages":{"$ref":"#/components/schemas/Map"},"reflectivelyExports":{"$ref":"#/components/schemas/WeakPairMap"},"layer":{"$ref":"#/components/schemas/ModuleLayer"},"ALL_UNNAMED_MODULE":{"$ref":"#/components/schemas/Module"},"reflectivelyUses":{"$ref":"#/components/schemas/WeakPairMap"},"reflectivelyReads":{"$ref":"#/components/schemas/WeakPairMap"},"name":{"type":"string"},"EVERYONE_SET":{"items":{"type":"object"},"type":"array","uniqueItems":true},"ALL_UNNAMED_MODULE_SET":{"items":{"type":"object"},"type":"array","uniqueItems":true}},"type":"object"},"InetAddress":{"name":"InetAddress","properties":{"cache":{"$ref":"#/components/schemas/ConcurrentMap"},"PREFER_IPV6_VALUE":{"format":"int32","type":"integer"},"IPv6":{"format":"int32","type":"integer"},"nameService":{"$ref":"#/components/schemas/NameService"},"canonicalHostName":{"type":"string"},"PREFER_IPV4_VALUE":{"format":"int32","type":"integer"},"IPv4":{"format":"int32","type":"integer"},"PREFER_SYSTEM_VALUE":{"format":"int32","type":"integer"},"holder":{"$ref":"#/components/schemas/InetAddressHolder"},"cachedLocalHost":{"$ref":"#/components/schemas/CachedLocalHost"},"FIELDS_OFFSET":{"format":"int64","type":"integer"},"serialPersistentFields":{"items":{"$ref":"#/components/schemas/ObjectStreamField"},"type":"array"},"impl":{"$ref":"#/components/schemas/InetAddressImpl"},"serialVersionUID":{"format":"int64","type":"integer"},"preferIPv6Address":{"format":"int32","type":"integer"},"UNSAFE":{"$ref":"#/components/schemas/Unsafe"},"expirySet":{"items":{"type":"object"},"type":"array","uniqueItems":true}},"type":"object"},"Map":{"name":"Map","type":"object"},"Key":{"name":"Key","type":"object"},"LangReflectAccess":{"name":"LangReflectAccess","type":"object"},"GroupTail":{"name":"GroupTail","properties":{"next":{"$ref":"#/components/schemas/Node"},"localIndex":{"format":"int32","type":"integer"},"groupIndex":{"format":"int32","type":"integer"}},"type":"object"},"CertificateFactory":{"name":"CertificateFactory","properties":{"provider":{"$ref":"#/components/schemas/Provider"},"type":{"type":"string"},"certFacSpi":{"$ref":"#/components/schemas/CertificateFactorySpi"}},"type":"object"},"Timestamp":{"name":"Timestamp","properties":{"myhash":{"format":"int32","type":"integer"},"serialVersionUID":{"format":"int64","type":"integer"},"signerCertPath":{"$ref":"#/components/schemas/CertPath"},"timestamp":{"$ref":"#/components/schemas/Date"}},"type":"object"},"URL":{"name":"URL","properties":{"userInfo":{"type":"string"},"handler":{"$ref":"#/components/schemas/URLStreamHandler"},"factory":{"$ref":"#/components/schemas/URLStreamHandlerFactory"},"query":{"type":"string"},"protocolPathProp":{"type":"string"},"serialPersistentFields":{"items":{"$ref":"#/components/schemas/ObjectStreamField"},"type":"array"},"path":{"type":"string"},"serialVersionUID":{"format":"int64","type":"integer"},"protocol":{"type":"string"},"ref":{"type":"string"},"file":{"type":"string"},"port":{"format":"int32","type":"integer"},"hashCode":{"format":"int32","type":"integer"},"defaultFactory":{"$ref":"#/components/schemas/URLStreamHandlerFactory"},"handlers":{"$ref":"#/components/schemas/Hashtable"},"authority":{"type":"string"},"host":{"type":"string"},"BUILTIN_HANDLERS_PREFIX":{"type":"string"},"hostAddress":{"$ref":"#/components/schemas/InetAddress"},"tempState":{"$ref":"#/components/schemas/UrlDeserializedState"},"gate":{"$ref":"#/components/schemas/ThreadLocal"},"streamHandlerLock":{"$ref":"#/components/schemas/Object"}},"type":"object"},"AnnotationType":{"name":"AnnotationType","properties":{"inherited":{"type":"boolean"},"members":{"$ref":"#/components/schemas/Map"},"memberDefaults":{"$ref":"#/components/schemas/Map"},"memberTypes":{"$ref":"#/components/schemas/Map"},"$assertionsDisabled":{"type":"boolean"},"retention":{"enums":["SOURCE","CLASS","RUNTIME"],"type":"string"}},"type":"object"},"Entry":{"name":"Entry","properties":{"next":{"$ref":"#/components/schemas/Entry"},"value":{"$ref":"#/components/schemas/Object"},"hash":{"format":"int32","type":"integer"},"key":{"$ref":"#/components/schemas/Object"}},"type":"object"},"WeakPairMap":{"name":"WeakPairMap","properties":{"map":{"$ref":"#/components/schemas/ConcurrentHashMap"},"queue":{"$ref":"#/components/schemas/ReferenceQueue"}},"type":"object"},"CertPath":{"name":"CertPath","properties":{"serialVersionUID":{"format":"int64","type":"integer"},"type":{"type":"string"}},"type":"object"},"CompositePrinterParser":{"name":"CompositePrinterParser","properties":{"optional":{"type":"boolean"},"printerParsers":{"items":{"$ref":"#/components/schemas/DateTimePrinterParser"},"type":"array"}},"type":"object"},"ConcurrentHashMap":{"name":"ConcurrentHashMap","properties":{"UNTREEIFY_THRESHOLD":{"format":"int32","type":"integer"},"TREEIFY_THRESHOLD":{"format":"int32","type":"integer"},"cellsBusy":{"format":"int32","type":"integer"},"values":{"$ref":"#/components/schemas/Collection"},"RESIZE_STAMP_SHIFT":{"format":"int32","type":"integer"},"DEFAULT_CONCURRENCY_LEVEL":{"format":"int32","type":"integer"},"HASH_BITS":{"format":"int32","type":"integer"},"baseCount":{"format":"int64","type":"integer"},"SIZECTL":{"format":"int64","type":"integer"},"CELLVALUE":{"format":"int64","type":"integer"},"serialVersionUID":{"format":"int64","type":"integer"},"U":{"$ref":"#/components/schemas/Unsafe"},"TRANSFERINDEX":{"format":"int64","type":"integer"},"transferIndex":{"format":"int32","type":"integer"},"LOAD_FACTOR":{"format":"float","type":"number"},"keySet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"DEFAULT_CAPACITY":{"format":"int32","type":"integer"},"NCPU":{"format":"int32","type":"integer"},"table":{"items":{"$ref":"#/components/schemas/Node"},"type":"array"},"ASHIFT":{"format":"int32","type":"integer"},"ABASE":{"format":"int32","type":"integer"},"sizeCtl":{"format":"int32","type":"integer"},"entrySet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"RESERVED":{"format":"int32","type":"integer"},"TREEBIN":{"format":"int32","type":"integer"},"MIN_TRANSFER_STRIDE":{"format":"int32","type":"integer"},"counterCells":{"items":{"$ref":"#/components/schemas/CounterCell"},"type":"array"},"serialPersistentFields":{"items":{"$ref":"#/components/schemas/ObjectStreamField"},"type":"array"},"MOVED":{"format":"int32","type":"integer"},"nextTable":{"items":{"$ref":"#/components/schemas/Node"},"type":"array"},"MAX_ARRAY_SIZE":{"format":"int32","type":"integer"},"MAX_RESIZERS":{"format":"int32","type":"integer"},"RESIZE_STAMP_BITS":{"format":"int32","type":"integer"},"BASECOUNT":{"format":"int64","type":"integer"},"MAXIMUM_CAPACITY":{"format":"int32","type":"integer"},"MIN_TREEIFY_CAPACITY":{"format":"int32","type":"integer"},"CELLSBUSY":{"format":"int64","type":"integer"}},"type":"object"},"MultiValueMap":{"name":"MultiValueMap","type":"object"},"CachedLocalHost":{"name":"CachedLocalHost","properties":{"host":{"type":"string"},"expiryTime":{"format":"int64","type":"integer"},"addr":{"$ref":"#/components/schemas/InetAddress"}},"type":"object"},"Pattern":{"name":"Pattern","properties":{"cursor":{"format":"int32","type":"integer"},"ALL_FLAGS":{"format":"int32","type":"integer"},"pattern":{"type":"string"},"flags":{"format":"int32","type":"integer"},"COMMENTS":{"format":"int32","type":"integer"},"namedGroups":{"$ref":"#/components/schemas/Map"},"localCount":{"format":"int32","type":"integer"},"groupNodes":{"items":{"$ref":"#/components/schemas/GroupHead"},"type":"array"},"hasGroupRef":{"type":"boolean"},"lastAccept":{"$ref":"#/components/schemas/Node"},"predicate":{"$ref":"#/components/schemas/CharPredicate"},"serialVersionUID":{"format":"int64","type":"integer"},"compiled":{"type":"boolean"},"normalizedPattern":{"type":"string"},"CASE_INSENSITIVE":{"format":"int32","type":"integer"},"root":{"$ref":"#/components/schemas/Node"},"buffer":{"items":{"format":"int32","type":"integer"},"type":"array"},"matchRoot":{"$ref":"#/components/schemas/Node"},"capturingGroupCount":{"format":"int32","type":"integer"},"lookbehindEnd":{"$ref":"#/components/schemas/Node"},"temp":{"items":{"format":"int32","type":"integer"},"type":"array"},"LITERAL":{"format":"int32","type":"integer"},"UNICODE_CHARACTER_CLASS":{"format":"int32","type":"integer"},"CANON_EQ":{"format":"int32","type":"integer"},"patternLength":{"format":"int32","type":"integer"},"localTCNCount":{"format":"int32","type":"integer"},"hasSupplementary":{"type":"boolean"},"$assertionsDisabled":{"type":"boolean"},"topClosureNodes":{"items":{"type":"object"},"type":"array"},"MAX_REPS":{"format":"int32","type":"integer"},"UNIX_LINES":{"format":"int32","type":"integer"},"accept":{"$ref":"#/components/schemas/Node"},"DOTALL":{"format":"int32","type":"integer"},"UNICODE_CASE":{"format":"int32","type":"integer"},"MULTILINE":{"format":"int32","type":"integer"}},"type":"object"},"UrlDeserializedState":{"name":"UrlDeserializedState","properties":{"protocol":{"type":"string"},"ref":{"type":"string"},"file":{"type":"string"},"port":{"format":"int32","type":"integer"},"hashCode":{"format":"int32","type":"integer"},"authority":{"type":"string"},"host":{"type":"string"}},"type":"object"},"DecimalFormatSymbols":{"name":"DecimalFormatSymbols","properties":{"exponential":{"type":"string"},"minusSign":{"type":"string"},"currencyInitialized":{"type":"boolean"},"decimalSeparator":{"type":"string"},"intlCurrencySymbol":{"type":"string"},"perMill":{"type":"string"},"currencySymbol":{"type":"string"},"monetarySeparator":{"type":"string"},"groupingSeparator":{"type":"string"},"locale":{"$ref":"#/components/schemas/Locale"},"percent":{"type":"string"},"currentSerialVersion":{"format":"int32","type":"integer"},"serialVersionUID":{"format":"int64","type":"integer"},"zeroDigit":{"type":"string"},"exponentialSeparator":{"type":"string"},"infinity":{"type":"string"},"NaN":{"type":"string"},"currency":{"$ref":"#/components/schemas/Currency"},"serialVersionOnStream":{"format":"int32","type":"integer"},"digit":{"type":"string"},"patternSeparator":{"type":"string"}},"type":"object"},"Hashtable":{"name":"Hashtable","properties":{"entrySet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"values":{"$ref":"#/components/schemas/Collection"},"count":{"format":"int32","type":"integer"},"threshold":{"format":"int32","type":"integer"},"modCount":{"format":"int32","type":"integer"},"serialVersionUID":{"format":"int64","type":"integer"},"MAX_ARRAY_SIZE":{"format":"int32","type":"integer"},"loadFactor":{"format":"float","type":"number"},"KEYS":{"format":"int32","type":"integer"},"VALUES":{"format":"int32","type":"integer"},"keySet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"table":{"items":{"$ref":"#/components/schemas/Entry"},"type":"array"},"ENTRIES":{"format":"int32","type":"integer"}},"type":"object"},"Constructor":{"name":"Constructor","properties":{"genericInfo":{"$ref":"#/components/schemas/ConstructorRepository"},"hasRealParameterData":{"type":"boolean"},"declaredAnnotations":{"$ref":"#/components/schemas/Map"},"printStackPropertiesSet":{"type":"boolean"},"parameterTypes":{"items":{"$ref":"#/components/schemas/Class"},"type":"array"},"signature":{"type":"string"},"annotations":{"items":{"format":"int32","type":"integer"},"type":"array"},"securityCheckCache":{"$ref":"#/components/schemas/Object"},"printStackWhenAccessFails":{"type":"boolean"},"slot":{"format":"int32","type":"integer"},"constructorAccessor":{"$ref":"#/components/schemas/ConstructorAccessor"},"modifiers":{"format":"int32","type":"integer"},"ACCESS_PERMISSION":{"$ref":"#/components/schemas/Permission"},"exceptionTypes":{"items":{"$ref":"#/components/schemas/Class"},"type":"array"},"root":{"$ref":"#/components/schemas/Constructor"},"parameterAnnotations":{"items":{"format":"int32","type":"integer"},"type":"array"},"override":{"type":"boolean"},"reflectionFactory":{"$ref":"#/components/schemas/ReflectionFactory"},"clazz":{"$ref":"#/components/schemas/Class"},"parameters":{"items":{"$ref":"#/components/schemas/Parameter"},"type":"array"}},"type":"object"},"Reference":{"name":"Reference","properties":{"next":{"$ref":"#/components/schemas/Reference"},"discovered":{"$ref":"#/components/schemas/Reference"},"referent":{"$ref":"#/components/schemas/Object"},"processPendingActive":{"type":"boolean"},"queue":{"$ref":"#/components/schemas/ReferenceQueue"},"processPendingLock":{"$ref":"#/components/schemas/Object"}},"type":"object"},"SocketPermission":{"name":"SocketPermission","properties":{"defaultDeny":{"type":"boolean"},"ALL":{"format":"int32","type":"integer"},"addresses":{"items":{"$ref":"#/components/schemas/InetAddress"},"type":"array"},"cname":{"type":"string"},"init_with_ip":{"type":"boolean"},"CONNECT":{"format":"int32","type":"integer"},"trustNameService":{"type":"boolean"},"wildcard":{"type":"boolean"},"serialVersionUID":{"format":"int64","type":"integer"},"PRIV_PORT_MAX":{"format":"int32","type":"integer"},"hostname":{"type":"string"},"PORT_MIN":{"format":"int32","type":"integer"},"hdomain":{"type":"string"},"LISTEN":{"format":"int32","type":"integer"},"cdomain":{"type":"string"},"NONE":{"format":"int32","type":"integer"},"DEF_EPH_LOW":{"format":"int32","type":"integer"},"mask":{"format":"int32","type":"integer"},"portrange":{"items":{"format":"int32","type":"integer"},"type":"array"},"debug":{"$ref":"#/components/schemas/Debug"},"PORT_MAX":{"format":"int32","type":"integer"},"debugInit":{"type":"boolean"},"ACCEPT":{"format":"int32","type":"integer"},"untrusted":{"type":"boolean"},"trusted":{"type":"boolean"},"RESOLVE":{"format":"int32","type":"integer"},"invalid":{"type":"boolean"},"name":{"type":"string"},"actions":{"type":"string"}},"type":"object"},"ModelMap":{"name":"ModelMap","properties":{"UNTREEIFY_THRESHOLD":{"format":"int32","type":"integer"},"TREEIFY_THRESHOLD":{"format":"int32","type":"integer"},"entrySet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"tail":{"$ref":"#/components/schemas/Entry"},"values":{"$ref":"#/components/schemas/Collection"},"DEFAULT_LOAD_FACTOR":{"format":"float","type":"number"},"threshold":{"format":"int32","type":"integer"},"DEFAULT_INITIAL_CAPACITY":{"format":"int32","type":"integer"},"head":{"$ref":"#/components/schemas/Entry"},"modCount":{"format":"int32","type":"integer"},"serialVersionUID":{"format":"int64","type":"integer"},"size":{"format":"int32","type":"integer"},"loadFactor":{"format":"float","type":"number"},"MAXIMUM_CAPACITY":{"format":"int32","type":"integer"},"keySet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"accessOrder":{"type":"boolean"},"MIN_TREEIFY_CAPACITY":{"format":"int32","type":"integer"},"table":{"items":{"$ref":"#/components/schemas/Node"},"type":"array"}},"type":"object"},"Lock":{"name":"Lock","type":"object"},"NameService":{"name":"NameService","type":"object"},"Type":{"name":"Type","type":"object"},"ResponseEntity":{"name":"ResponseEntity","properties":{"headers":{"$ref":"#/components/schemas/HttpHeaders"},"body":{"$ref":"#/components/schemas/Object"},"EMPTY":{"$ref":"#/components/schemas/HttpEntity"},"status":{"$ref":"#/components/schemas/Object"}},"type":"object"},"Parameter":{"name":"Parameter","properties":{"declaredAnnotations":{"$ref":"#/components/schemas/Map"},"parameterTypeCache":{"$ref":"#/components/schemas/Type"},"parameterClassCache":{"$ref":"#/components/schemas/Class"},"name":{"type":"string"},"index":{"format":"int32","type":"integer"},"modifiers":{"format":"int32","type":"integer"},"executable":{"$ref":"#/components/schemas/Executable"}},"type":"object"},"Principal":{"name":"Principal","type":"object"},"BaseLocale":{"name":"BaseLocale","properties":{"variant":{"type":"string"},"language":{"type":"string"},"CACHE":{"$ref":"#/components/schemas/Cache"},"region":{"type":"string"},"script":{"type":"string"},"hash":{"format":"int32","type":"integer"},"SEP":{"type":"string"}},"type":"object"},"Locale":{"name":"Locale","properties":{"PRC":{"$ref":"#/components/schemas/Locale"},"LOCALECACHE":{"$ref":"#/components/schemas/Cache"},"CANADA":{"$ref":"#/components/schemas/Locale"},"ROOT":{"$ref":"#/components/schemas/Locale"},"TAIWAN":{"$ref":"#/components/schemas/Locale"},"baseLocale":{"$ref":"#/components/schemas/BaseLocale"},"CHINESE":{"$ref":"#/components/schemas/Locale"},"DISPLAY_LANGUAGE":{"format":"int32","type":"integer"},"languageTag":{"type":"string"},"KOREA":{"$ref":"#/components/schemas/Locale"},"hashCodeValue":{"format":"int32","type":"integer"},"FRENCH":{"$ref":"#/components/schemas/Locale"},"FRANCE":{"$ref":"#/components/schemas/Locale"},"TRADITIONAL_CHINESE":{"$ref":"#/components/schemas/Locale"},"serialVersionUID":{"format":"int64","type":"integer"},"GERMANY":{"$ref":"#/components/schemas/Locale"},"DISPLAY_COUNTRY":{"format":"int32","type":"integer"},"ITALIAN":{"$ref":"#/components/schemas/Locale"},"PRIVATE_USE_EXTENSION":{"type":"string"},"defaultDisplayLocale":{"$ref":"#/components/schemas/Locale"},"UK":{"$ref":"#/components/schemas/Locale"},"JAPAN":{"$ref":"#/components/schemas/Locale"},"UNICODE_LOCALE_EXTENSION":{"type":"string"},"SIMPLIFIED_CHINESE":{"$ref":"#/components/schemas/Locale"},"DISPLAY_VARIANT":{"format":"int32","type":"integer"},"GERMAN":{"$ref":"#/components/schemas/Locale"},"US":{"$ref":"#/components/schemas/Locale"},"ENGLISH":{"$ref":"#/components/schemas/Locale"},"CHINA":{"$ref":"#/components/schemas/Locale"},"ITALY":{"$ref":"#/components/schemas/Locale"},"defaultFormatLocale":{"$ref":"#/components/schemas/Locale"},"$assertionsDisabled":{"type":"boolean"},"isoCountries":{"items":{"type":"string"},"type":"array"},"CANADA_FRENCH":{"$ref":"#/components/schemas/Locale"},"defaultLocale":{"$ref":"#/components/schemas/Locale"},"serialPersistentFields":{"items":{"$ref":"#/components/schemas/ObjectStreamField"},"type":"array"},"localeExtensions":{"$ref":"#/components/schemas/LocaleExtensions"},"isoLanguages":{"items":{"type":"string"},"type":"array"},"JAPANESE":{"$ref":"#/components/schemas/Locale"},"DISPLAY_SCRIPT":{"format":"int32","type":"integer"},"KOREAN":{"$ref":"#/components/schemas/Locale"}},"type":"object"},"ProtectionDomain":{"name":"ProtectionDomain","properties":{"staticPermissions":{"type":"boolean"},"hasAllPerm":{"type":"boolean"},"codesource":{"$ref":"#/components/schemas/CodeSource"},"permissions":{"$ref":"#/components/schemas/PermissionCollection"},"classloader":{"$ref":"#/components/schemas/ClassLoader"},"filePermCompatInPD":{"type":"boolean"},"principals":{"items":{"$ref":"#/components/schemas/Principal"},"type":"array"},"key":{"$ref":"#/components/schemas/Key"}},"type":"object"},"ConstructorRepository":{"name":"ConstructorRepository","properties":{"factory":{"$ref":"#/components/schemas/GenericsFactory"},"exceptionTypes":{"items":{"$ref":"#/components/schemas/Type"},"type":"array"},"parameterTypes":{"items":{"$ref":"#/components/schemas/Type"},"type":"array"},"tree":{"$ref":"#/components/schemas/Tree"},"typeParameters":{"items":{"$ref":"#/components/schemas/TypeVariable"},"type":"array"}},"type":"object"},"ClassRepository":{"name":"ClassRepository","properties":{"factory":{"$ref":"#/components/schemas/GenericsFactory"},"superclass":{"$ref":"#/components/schemas/Type"},"tree":{"$ref":"#/components/schemas/Tree"},"NONE":{"$ref":"#/components/schemas/ClassRepository"},"superInterfaces":{"items":{"$ref":"#/components/schemas/Type"},"type":"array"},"typeParameters":{"items":{"$ref":"#/components/schemas/TypeVariable"},"type":"array"}},"type":"object"},"ServiceKey":{"name":"ServiceKey","properties":{"originalAlgorithm":{"type":"string"},"type":{"type":"string"},"algorithm":{"type":"string"}},"type":"object"},"Unsafe":{"name":"Unsafe","properties":{"ARRAY_BOOLEAN_BASE_OFFSET":{"format":"int32","type":"integer"},"ARRAY_CHAR_BASE_OFFSET":{"format":"int32","type":"integer"},"BE":{"type":"boolean"},"ARRAY_SHORT_BASE_OFFSET":{"format":"int32","type":"integer"},"ARRAY_INT_BASE_OFFSET":{"format":"int32","type":"integer"},"INVALID_FIELD_OFFSET":{"format":"int32","type":"integer"},"ARRAY_OBJECT_INDEX_SCALE":{"format":"int32","type":"integer"},"ADDRESS_SIZE":{"format":"int32","type":"integer"},"ARRAY_LONG_INDEX_SCALE":{"format":"int32","type":"integer"},"ARRAY_FLOAT_INDEX_SCALE":{"format":"int32","type":"integer"},"ARRAY_DOUBLE_INDEX_SCALE":{"format":"int32","type":"integer"},"unalignedAccess":{"type":"boolean"},"ARRAY_FLOAT_BASE_OFFSET":{"format":"int32","type":"integer"},"ARRAY_DOUBLE_BASE_OFFSET":{"format":"int32","type":"integer"},"theUnsafe":{"$ref":"#/components/schemas/Unsafe"},"ARRAY_BOOLEAN_INDEX_SCALE":{"format":"int32","type":"integer"},"ARRAY_CHAR_INDEX_SCALE":{"format":"int32","type":"integer"},"ARRAY_INT_INDEX_SCALE":{"format":"int32","type":"integer"},"ARRAY_BYTE_INDEX_SCALE":{"format":"int32","type":"integer"},"ARRAY_LONG_BASE_OFFSET":{"format":"int32","type":"integer"},"ARRAY_OBJECT_BASE_OFFSET":{"format":"int32","type":"integer"},"ARRAY_SHORT_INDEX_SCALE":{"format":"int32","type":"integer"},"ARRAY_BYTE_BASE_OFFSET":{"format":"int32","type":"integer"}},"type":"object"},"CalendarDate":{"name":"CalendarDate","properties":{"zoneOffset":{"format":"int32","type":"integer"},"hours":{"format":"int32","type":"integer"},"zoneinfo":{"$ref":"#/components/schemas/TimeZone"},"year":{"format":"int32","type":"integer"},"minutes":{"format":"int32","type":"integer"},"normalized":{"type":"boolean"},"forceStandardTime":{"type":"boolean"},"FIELD_UNDEFINED":{"format":"int32","type":"integer"},"leapYear":{"type":"boolean"},"locale":{"$ref":"#/components/schemas/Locale"},"fraction":{"format":"int64","type":"integer"},"TIME_UNDEFINED":{"format":"int64","type":"integer"},"seconds":{"format":"int32","type":"integer"},"dayOfWeek":{"format":"int32","type":"integer"},"month":{"format":"int32","type":"integer"},"daylightSaving":{"format":"int32","type":"integer"},"era":{"$ref":"#/components/schemas/Era"},"dayOfMonth":{"format":"int32","type":"integer"},"millis":{"format":"int32","type":"integer"}},"type":"object"},"CounterCell":{"name":"CounterCell","properties":{"value":{"format":"int64","type":"integer"}},"type":"object"},"Version":{"name":"Version","properties":{"sequence":{"items":{"type":"object"},"type":"array"},"pre":{"items":{"type":"object"},"type":"array"},"build":{"items":{"type":"object"},"type":"array"},"version":{"type":"string"}},"type":"object"},"ReflectionFactory":{"name":"ReflectionFactory","properties":{"initted":{"type":"boolean"},"inflationThreshold":{"format":"int32","type":"integer"},"soleInstance":{"$ref":"#/components/schemas/ReflectionFactory"},"reflectionFactoryAccessPerm":{"$ref":"#/components/schemas/Permission"},"langReflectAccess":{"$ref":"#/components/schemas/LangReflectAccess"},"hasStaticInitializerMethod":{"$ref":"#/components/schemas/Method"},"noInflation":{"type":"boolean"}},"type":"object"},"Tree":{"name":"Tree","type":"object"},"Executable":{"name":"Executable","properties":{"hasRealParameterData":{"type":"boolean"},"declaredAnnotations":{"$ref":"#/components/schemas/Map"},"printStackPropertiesSet":{"type":"boolean"},"securityCheckCache":{"$ref":"#/components/schemas/Object"},"printStackWhenAccessFails":{"type":"boolean"},"override":{"type":"boolean"},"reflectionFactory":{"$ref":"#/components/schemas/ReflectionFactory"},"parameters":{"items":{"$ref":"#/components/schemas/Parameter"},"type":"array"},"ACCESS_PERMISSION":{"$ref":"#/components/schemas/Permission"}},"type":"object"},"Foo":{"name":"Foo","properties":{"l1":{"format":"int64","type":"integer"},"l2":{"items":{"format":"int64","type":"integer"},"type":"array"},"foo":{"$ref":"#/components/schemas/Foo"},"f1":{"format":"float","type":"number"},"f2":{"items":{"format":"float","type":"number"},"type":"array"},"d1":{"format":"double","type":"number"},"d2":{"items":{"format":"double","type":"number"},"type":"array"},"b1":{"type":"boolean"},"b2":{"items":{"type":"boolean"},"type":"array"},"ss1":{"items":{"type":"object"},"type":"array","uniqueItems":true},"bar2":{"items":{"$ref":"#/components/schemas/Bar"},"type":"array"},"bar":{"$ref":"#/components/schemas/Bar"},"foo2":{"items":{"$ref":"#/components/schemas/Foo"},"type":"array"},"s1":{"format":"int32","type":"integer"},"s2":{"items":{"format":"int32","type":"integer"},"type":"array"},"m1":{"$ref":"#/components/schemas/Map"},"k1":{"type":"string"},"k2":{"items":{"type":"string"},"type":"array"},"k3":{"items":{"type":"object"},"type":"array"},"i1":{"format":"int32","type":"integer"},"i2":{"items":{"format":"int32","type":"integer"},"type":"array"},"enumnumnum":{"enums":["A","B","C"],"type":"string"},"c1":{"type":"string"},"c2":{"items":{"type":"string"},"type":"array"},"list1":{"items":{"type":"object"},"type":"array"},"list4":{"items":{"type":"object"},"type":"array"},"list3":{"items":{"type":"object"},"type":"array"},"list2":{"items":{"type":"object"},"type":"array"}},"type":"object"},"ConcurrentMap":{"name":"ConcurrentMap","type":"object"},"ValuesView":{"name":"ValuesView","properties":{"serialVersionUID":{"format":"int64","type":"integer"},"OOME_MSG":{"type":"string"},"map":{"$ref":"#/components/schemas/ConcurrentHashMap"}},"type":"object"},"MethodRepository":{"name":"MethodRepository","properties":{"factory":{"$ref":"#/components/schemas/GenericsFactory"},"exceptionTypes":{"items":{"$ref":"#/components/schemas/Type"},"type":"array"},"parameterTypes":{"items":{"$ref":"#/components/schemas/Type"},"type":"array"},"tree":{"$ref":"#/components/schemas/Tree"},"typeParameters":{"items":{"$ref":"#/components/schemas/TypeVariable"},"type":"array"},"returnType":{"$ref":"#/components/schemas/Type"}},"type":"object"},"ObjectStreamField":{"name":"ObjectStreamField","properties":{"field":{"$ref":"#/components/schemas/Field"},"offset":{"format":"int32","type":"integer"},"signature":{"type":"string"},"unshared":{"type":"boolean"},"name":{"type":"string"},"type":{"$ref":"#/components/schemas/Class"}},"type":"object"},"GenericsFactory":{"name":"GenericsFactory","type":"object"},"Collection":{"name":"Collection","type":"object"},"Class":{"name":"Class","properties":{"genericInfo":{"$ref":"#/components/schemas/ClassRepository"},"annotationData":{"$ref":"#/components/schemas/AnnotationData"},"ENUM":{"format":"int32","type":"integer"},"componentType":{"$ref":"#/components/schemas/Class"},"enumConstantDirectory":{"$ref":"#/components/schemas/Map"},"classRedefinedCount":{"format":"int32","type":"integer"},"cachedConstructor":{"$ref":"#/components/schemas/Constructor"},"module":{"$ref":"#/components/schemas/Module"},"SYNTHETIC":{"format":"int32","type":"integer"},"annotationType":{"$ref":"#/components/schemas/AnnotationType"},"newInstanceCallerCache":{"$ref":"#/components/schemas/Class"},"reflectionData":{"$ref":"#/components/schemas/SoftReference"},"classValueMap":{"$ref":"#/components/schemas/ClassValueMap"},"serialPersistentFields":{"items":{"$ref":"#/components/schemas/ObjectStreamField"},"type":"array"},"serialVersionUID":{"format":"int64","type":"integer"},"ANNOTATION":{"format":"int32","type":"integer"},"enumConstants":{"items":{"$ref":"#/components/schemas/Object"},"type":"array"},"name":{"type":"string"},"packageName":{"type":"string"},"reflectionFactory":{"$ref":"#/components/schemas/ReflectionFactory"},"allPermDomain":{"$ref":"#/components/schemas/ProtectionDomain"},"EMPTY_CLASS_ARRAY":{"items":{"$ref":"#/components/schemas/Class"},"type":"array"}},"type":"object"},"InetAddressImpl":{"name":"InetAddressImpl","type":"object"},"DateTimeFormatter":{"name":"DateTimeFormatter","properties":{"printerParser":{"$ref":"#/components/schemas/CompositePrinterParser"},"ISO_OFFSET_TIME":{"$ref":"#/components/schemas/DateTimeFormatter"},"resolverFields":{"items":{"type":"object"},"type":"array","uniqueItems":true},"ISO_LOCAL_DATE_TIME":{"$ref":"#/components/schemas/DateTimeFormatter"},"PARSED_EXCESS_DAYS":{"$ref":"#/components/schemas/TemporalQuery"},"decimalStyle":{"$ref":"#/components/schemas/DecimalStyle"},"RFC_1123_DATE_TIME":{"$ref":"#/components/schemas/DateTimeFormatter"},"ISO_INSTANT":{"$ref":"#/components/schemas/DateTimeFormatter"},"ISO_ZONED_DATE_TIME":{"$ref":"#/components/schemas/DateTimeFormatter"},"ISO_OFFSET_DATE_TIME":{"$ref":"#/components/schemas/DateTimeFormatter"},"ISO_DATE_TIME":{"$ref":"#/components/schemas/DateTimeFormatter"},"locale":{"$ref":"#/components/schemas/Locale"},"ISO_DATE":{"$ref":"#/components/schemas/DateTimeFormatter"},"ISO_TIME":{"$ref":"#/components/schemas/DateTimeFormatter"},"ISO_LOCAL_TIME":{"$ref":"#/components/schemas/DateTimeFormatter"},"ISO_OFFSET_DATE":{"$ref":"#/components/schemas/DateTimeFormatter"},"zone":{"$ref":"#/components/schemas/ZoneId"},"ISO_ORDINAL_DATE":{"$ref":"#/components/schemas/DateTimeFormatter"},"resolverStyle":{"enums":["STRICT","SMART","LENIENT"],"type":"string"},"ISO_LOCAL_DATE":{"$ref":"#/components/schemas/DateTimeFormatter"},"chrono":{"$ref":"#/components/schemas/Chronology"},"BASIC_ISO_DATE":{"$ref":"#/components/schemas/DateTimeFormatter"},"ISO_WEEK_DATE":{"$ref":"#/components/schemas/DateTimeFormatter"},"PARSED_LEAP_SECOND":{"$ref":"#/components/schemas/TemporalQuery"}},"type":"object"},"Certificate":{"name":"Certificate","properties":{"serialVersionUID":{"format":"int64","type":"integer"},"type":{"type":"string"},"hash":{"format":"int32","type":"integer"}},"type":"object"},"Cache":{"name":"Cache","properties":{"map":{"$ref":"#/components/schemas/ConcurrentMap"},"queue":{"$ref":"#/components/schemas/ReferenceQueue"}},"type":"object"},"ConstructorAccessor":{"name":"ConstructorAccessor","type":"object"},"PermissionCollection":{"name":"PermissionCollection","properties":{"serialVersionUID":{"format":"int64","type":"integer"},"readOnly":{"type":"boolean"}},"type":"object"},"HttpEntity":{"name":"HttpEntity","properties":{"headers":{"$ref":"#/components/schemas/HttpHeaders"},"body":{"$ref":"#/components/schemas/Object"},"EMPTY":{"$ref":"#/components/schemas/HttpEntity"}},"type":"object"},"AnnotationData":{"name":"AnnotationData","properties":{"declaredAnnotations":{"$ref":"#/components/schemas/Map"},"redefinedCount":{"format":"int32","type":"integer"},"annotations":{"$ref":"#/components/schemas/Map"}},"type":"object"},"ModelAndView":{"name":"ModelAndView","properties":{"view":{"$ref":"#/components/schemas/Object"},"model":{"$ref":"#/components/schemas/ModelMap"},"cleared":{"type":"boolean"},"status":{"enums":["100 CONTINUE","101 SWITCHING_PROTOCOLS","102 PROCESSING","103 CHECKPOINT","200 OK","201 CREATED","202 ACCEPTED","203 NON_AUTHORITATIVE_INFORMATION","204 NO_CONTENT","205 RESET_CONTENT","206 PARTIAL_CONTENT","207 MULTI_STATUS","208 ALREADY_REPORTED","226 IM_USED","300 MULTIPLE_CHOICES","301 MOVED_PERMANENTLY","302 FOUND","302 MOVED_TEMPORARILY","303 SEE_OTHER","304 NOT_MODIFIED","305 USE_PROXY","307 TEMPORARY_REDIRECT","308 PERMANENT_REDIRECT","400 BAD_REQUEST","401 UNAUTHORIZED","402 PAYMENT_REQUIRED","403 FORBIDDEN","404 NOT_FOUND","405 METHOD_NOT_ALLOWED","406 NOT_ACCEPTABLE","407 PROXY_AUTHENTICATION_REQUIRED","408 REQUEST_TIMEOUT","409 CONFLICT","410 GONE","411 LENGTH_REQUIRED","412 PRECONDITION_FAILED","413 PAYLOAD_TOO_LARGE","413 REQUEST_ENTITY_TOO_LARGE","414 URI_TOO_LONG","414 REQUEST_URI_TOO_LONG","415 UNSUPPORTED_MEDIA_TYPE","416 REQUESTED_RANGE_NOT_SATISFIABLE","417 EXPECTATION_FAILED","418 I_AM_A_TEAPOT","419 INSUFFICIENT_SPACE_ON_RESOURCE","420 METHOD_FAILURE","421 DESTINATION_LOCKED","422 UNPROCESSABLE_ENTITY","423 LOCKED","424 FAILED_DEPENDENCY","425 TOO_EARLY","426 UPGRADE_REQUIRED","428 PRECONDITION_REQUIRED","429 TOO_MANY_REQUESTS","431 REQUEST_HEADER_FIELDS_TOO_LARGE","451 UNAVAILABLE_FOR_LEGAL_REASONS","500 INTERNAL_SERVER_ERROR","501 NOT_IMPLEMENTED","502 BAD_GATEWAY","503 SERVICE_UNAVAILABLE","504 GATEWAY_TIMEOUT","505 HTTP_VERSION_NOT_SUPPORTED","506 VARIANT_ALSO_NEGOTIATES","507 INSUFFICIENT_STORAGE","508 LOOP_DETECTED","509 BANDWIDTH_LIMIT_EXCEEDED","510 NOT_EXTENDED","511 NETWORK_AUTHENTICATION_REQUIRED"],"type":"string"}},"type":"object"},"URLStreamHandlerFactory":{"name":"URLStreamHandlerFactory","type":"object"},"Properties":{"name":"Properties","properties":{"entrySet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"values":{"$ref":"#/components/schemas/Collection"},"count":{"format":"int32","type":"integer"},"threshold":{"format":"int32","type":"integer"},"modCount":{"format":"int32","type":"integer"},"serialVersionUID":{"format":"int64","type":"integer"},"MAX_ARRAY_SIZE":{"format":"int32","type":"integer"},"defaults":{"$ref":"#/components/schemas/Properties"},"loadFactor":{"format":"float","type":"number"},"hexDigit":{"items":{"type":"string"},"type":"array"},"KEYS":{"format":"int32","type":"integer"},"VALUES":{"format":"int32","type":"integer"},"map":{"$ref":"#/components/schemas/ConcurrentHashMap"},"keySet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"table":{"items":{"$ref":"#/components/schemas/Entry"},"type":"array"},"ENTRIES":{"format":"int32","type":"integer"}},"type":"object"},"InetAddressHolder":{"name":"InetAddressHolder","properties":{"originalHostName":{"type":"string"},"hostName":{"type":"string"},"address":{"format":"int32","type":"integer"},"family":{"format":"int32","type":"integer"}},"type":"object"},"FieldRepository":{"name":"FieldRepository","properties":{"factory":{"$ref":"#/components/schemas/GenericsFactory"},"tree":{"$ref":"#/components/schemas/Tree"},"genericType":{"$ref":"#/components/schemas/Type"}},"type":"object"},"FieldAccessor":{"name":"FieldAccessor","type":"object"},"ClassValueMap":{"name":"ClassValueMap","properties":{"INITIAL_ENTRIES":{"format":"int32","type":"integer"},"cacheArray":{"items":{"$ref":"#/components/schemas/Entry"},"type":"array"},"entrySet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"cacheLoadLimit":{"format":"int32","type":"integer"},"values":{"$ref":"#/components/schemas/Collection"},"DEFAULT_LOAD_FACTOR":{"format":"float","type":"number"},"CACHE_LOAD_LIMIT":{"format":"int32","type":"integer"},"PROBE_LIMIT":{"format":"int32","type":"integer"},"$assertionsDisabled":{"type":"boolean"},"threshold":{"format":"int32","type":"integer"},"DEFAULT_INITIAL_CAPACITY":{"format":"int32","type":"integer"},"modCount":{"format":"int32","type":"integer"},"size":{"format":"int32","type":"integer"},"loadFactor":{"format":"float","type":"number"},"cacheLoad":{"format":"int32","type":"integer"},"MAXIMUM_CAPACITY":{"format":"int32","type":"integer"},"keySet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"table":{"items":{"$ref":"#/components/schemas/Entry"},"type":"array"},"queue":{"$ref":"#/components/schemas/ReferenceQueue"},"NULL_KEY":{"$ref":"#/components/schemas/Object"}},"type":"object"},"Field":{"name":"Field","properties":{"genericInfo":{"$ref":"#/components/schemas/FieldRepository"},"declaredAnnotations":{"$ref":"#/components/schemas/Map"},"printStackPropertiesSet":{"type":"boolean"},"overrideFieldAccessor":{"$ref":"#/components/schemas/FieldAccessor"},"signature":{"type":"string"},"annotations":{"items":{"format":"int32","type":"integer"},"type":"array"},"securityCheckCache":{"$ref":"#/components/schemas/Object"},"printStackWhenAccessFails":{"type":"boolean"},"slot":{"format":"int32","type":"integer"},"type":{"$ref":"#/components/schemas/Class"},"modifiers":{"format":"int32","type":"integer"},"fieldAccessor":{"$ref":"#/components/schemas/FieldAccessor"},"ACCESS_PERMISSION":{"$ref":"#/components/schemas/Permission"},"root":{"$ref":"#/components/schemas/Field"},"name":{"type":"string"},"override":{"type":"boolean"},"reflectionFactory":{"$ref":"#/components/schemas/ReflectionFactory"},"clazz":{"$ref":"#/components/schemas/Class"}},"type":"object"},"LocaleExtensions":{"name":"LocaleExtensions","properties":{"NUMBER_THAI":{"$ref":"#/components/schemas/LocaleExtensions"},"extensionMap":{"$ref":"#/components/schemas/Map"},"CALENDAR_JAPANESE":{"$ref":"#/components/schemas/LocaleExtensions"},"$assertionsDisabled":{"type":"boolean"},"id":{"type":"string"}},"type":"object"},"DateTimePrinterParser":{"name":"DateTimePrinterParser","type":"object"},"GroupHead":{"name":"GroupHead","properties":{"next":{"$ref":"#/components/schemas/Node"},"tail":{"$ref":"#/components/schemas/GroupTail"},"localIndex":{"format":"int32","type":"integer"}},"type":"object"},"ClassLoader":{"name":"ClassLoader","properties":{"parent":{"$ref":"#/components/schemas/ClassLoader"},"parallelLockMap":{"$ref":"#/components/schemas/ConcurrentHashMap"},"unnamedModule":{"$ref":"#/components/schemas/Module"},"packageAssertionStatus":{"$ref":"#/components/schemas/Map"},"classLoaderValueMap":{"$ref":"#/components/schemas/ConcurrentHashMap"},"classes":{"items":{"type":"object"},"type":"array"},"$assertionsDisabled":{"type":"boolean"},"usr_paths":{"items":{"type":"string"},"type":"array"},"defaultDomain":{"$ref":"#/components/schemas/ProtectionDomain"},"packages":{"$ref":"#/components/schemas/ConcurrentHashMap"},"nativeLibraryContext":{"items":{"type":"object"},"type":"array"},"sys_paths":{"items":{"type":"string"},"type":"array"},"systemNativeLibraries":{"items":{"type":"object"},"type":"array"},"classAssertionStatus":{"$ref":"#/components/schemas/Map"},"package2certs":{"$ref":"#/components/schemas/Map"},"nocerts":{"items":{"$ref":"#/components/schemas/Certificate"},"type":"array"},"nativeLibraries":{"items":{"type":"object"},"type":"array"},"name":{"type":"string"},"defaultAssertionStatus":{"type":"boolean"},"assertionLock":{"$ref":"#/components/schemas/Object"},"loadedLibraryNames":{"items":{"type":"object"},"type":"array"},"scl":{"$ref":"#/components/schemas/ClassLoader"}},"type":"object"},"Chronology":{"name":"Chronology","type":"object"},"CodeSigner":{"name":"CodeSigner","properties":{"myhash":{"format":"int32","type":"integer"},"serialVersionUID":{"format":"int64","type":"integer"},"signerCertPath":{"$ref":"#/components/schemas/CertPath"},"timestamp":{"$ref":"#/components/schemas/Timestamp"}},"type":"object"},"Debug":{"name":"Debug","properties":{"args":{"type":"string"},"hexDigits":{"items":{"type":"string"},"type":"array"},"prefix":{"type":"string"}},"type":"object"},"BaseCalendar":{"name":"BaseCalendar","properties":{"GREGORIAN_INSTANCE":{"$ref":"#/components/schemas/Gregorian"},"THURSDAY":{"format":"int32","type":"integer"},"MAY":{"format":"int32","type":"integer"},"DECEMBER":{"format":"int32","type":"integer"},"FRIDAY":{"format":"int32","type":"integer"},"FEBRUARY":{"format":"int32","type":"integer"},"WEDNESDAY":{"format":"int32","type":"integer"},"EPOCH_OFFSET":{"format":"int32","type":"integer"},"DAYS_IN_MONTH":{"items":{"format":"int32","type":"integer"},"type":"array"},"DAY_IN_MILLIS":{"format":"int32","type":"integer"},"SATURDAY":{"format":"int32","type":"integer"},"ACCUMULATED_DAYS_IN_MONTH":{"items":{"format":"int32","type":"integer"},"type":"array"},"initialized":{"type":"boolean"},"APRIL":{"format":"int32","type":"integer"},"JANUARY":{"format":"int32","type":"integer"},"JUNE":{"format":"int32","type":"integer"},"namePairs":{"items":{"type":"string"},"type":"array"},"SUNDAY":{"format":"int32","type":"integer"},"MINUTE_IN_MILLIS":{"format":"int32","type":"integer"},"OCTOBER":{"format":"int32","type":"integer"},"TUESDAY":{"format":"int32","type":"integer"},"FIXED_DATES":{"items":{"format":"int32","type":"integer"},"type":"array"},"$assertionsDisabled":{"type":"boolean"},"SEPTEMBER":{"format":"int32","type":"integer"},"NOVEMBER":{"format":"int32","type":"integer"},"HOUR_IN_MILLIS":{"format":"int32","type":"integer"},"eras":{"items":{"$ref":"#/components/schemas/Era"},"type":"array"},"MONDAY":{"format":"int32","type":"integer"},"ACCUMULATED_DAYS_IN_MONTH_LEAP":{"items":{"format":"int32","type":"integer"},"type":"array"},"SECOND_IN_MILLIS":{"format":"int32","type":"integer"},"BASE_YEAR":{"format":"int32","type":"integer"},"names":{"$ref":"#/components/schemas/ConcurrentMap"},"calendars":{"$ref":"#/components/schemas/ConcurrentMap"},"MARCH":{"format":"int32","type":"integer"},"AUGUST":{"format":"int32","type":"integer"},"JULY":{"format":"int32","type":"integer"},"PACKAGE_NAME":{"type":"string"}},"type":"object"},"ZoneId":{"name":"ZoneId","properties":{"serialVersionUID":{"format":"int64","type":"integer"},"SHORT_IDS":{"$ref":"#/components/schemas/Map"}},"type":"object"},"ReferenceQueue":{"name":"ReferenceQueue","properties":{"head":{"$ref":"#/components/schemas/Reference"},"ENQUEUED":{"$ref":"#/components/schemas/ReferenceQueue"},"queueLength":{"format":"int64","type":"integer"},"NULL":{"$ref":"#/components/schemas/ReferenceQueue"},"lock":{"$ref":"#/components/schemas/Lock"},"$assertionsDisabled":{"type":"boolean"}},"type":"object"},"Gregorian":{"name":"Gregorian","properties":{"GREGORIAN_INSTANCE":{"$ref":"#/components/schemas/Gregorian"},"THURSDAY":{"format":"int32","type":"integer"},"MAY":{"format":"int32","type":"integer"},"DECEMBER":{"format":"int32","type":"integer"},"FRIDAY":{"format":"int32","type":"integer"},"FEBRUARY":{"format":"int32","type":"integer"},"WEDNESDAY":{"format":"int32","type":"integer"},"EPOCH_OFFSET":{"format":"int32","type":"integer"},"DAYS_IN_MONTH":{"items":{"format":"int32","type":"integer"},"type":"array"},"DAY_IN_MILLIS":{"format":"int32","type":"integer"},"SATURDAY":{"format":"int32","type":"integer"},"ACCUMULATED_DAYS_IN_MONTH":{"items":{"format":"int32","type":"integer"},"type":"array"},"initialized":{"type":"boolean"},"APRIL":{"format":"int32","type":"integer"},"JANUARY":{"format":"int32","type":"integer"},"JUNE":{"format":"int32","type":"integer"},"namePairs":{"items":{"type":"string"},"type":"array"},"SUNDAY":{"format":"int32","type":"integer"},"MINUTE_IN_MILLIS":{"format":"int32","type":"integer"},"OCTOBER":{"format":"int32","type":"integer"},"TUESDAY":{"format":"int32","type":"integer"},"FIXED_DATES":{"items":{"format":"int32","type":"integer"},"type":"array"},"$assertionsDisabled":{"type":"boolean"},"SEPTEMBER":{"format":"int32","type":"integer"},"NOVEMBER":{"format":"int32","type":"integer"},"HOUR_IN_MILLIS":{"format":"int32","type":"integer"},"eras":{"items":{"$ref":"#/components/schemas/Era"},"type":"array"},"MONDAY":{"format":"int32","type":"integer"},"ACCUMULATED_DAYS_IN_MONTH_LEAP":{"items":{"format":"int32","type":"integer"},"type":"array"},"SECOND_IN_MILLIS":{"format":"int32","type":"integer"},"BASE_YEAR":{"format":"int32","type":"integer"},"names":{"$ref":"#/components/schemas/ConcurrentMap"},"calendars":{"$ref":"#/components/schemas/ConcurrentMap"},"MARCH":{"format":"int32","type":"integer"},"AUGUST":{"format":"int32","type":"integer"},"JULY":{"format":"int32","type":"integer"},"PACKAGE_NAME":{"type":"string"}},"type":"object"},"Date":{"name":"Date","properties":{"zoneOffset":{"format":"int32","type":"integer"},"cachedFixedDateJan1":{"format":"int64","type":"integer"},"hours":{"format":"int32","type":"integer"},"zoneinfo":{"$ref":"#/components/schemas/TimeZone"},"year":{"format":"int32","type":"integer"},"minutes":{"format":"int32","type":"integer"},"normalized":{"type":"boolean"},"forceStandardTime":{"type":"boolean"},"FIELD_UNDEFINED":{"format":"int32","type":"integer"},"cachedYear":{"format":"int32","type":"integer"},"leapYear":{"type":"boolean"},"locale":{"$ref":"#/components/schemas/Locale"},"fraction":{"format":"int64","type":"integer"},"TIME_UNDEFINED":{"format":"int64","type":"integer"},"cachedFixedDateNextJan1":{"format":"int64","type":"integer"},"seconds":{"format":"int32","type":"integer"},"dayOfWeek":{"format":"int32","type":"integer"},"month":{"format":"int32","type":"integer"},"daylightSaving":{"format":"int32","type":"integer"},"era":{"$ref":"#/components/schemas/Era"},"dayOfMonth":{"format":"int32","type":"integer"},"millis":{"format":"int32","type":"integer"}},"type":"object"},"Provider":{"name":"Provider","properties":{"values":{"$ref":"#/components/schemas/Collection"},"threshold":{"format":"int32","type":"integer"},"legacyStrings":{"$ref":"#/components/schemas/Map"},"modCount":{"format":"int32","type":"integer"},"serialVersionUID":{"format":"int64","type":"integer"},"serviceMap":{"$ref":"#/components/schemas/Map"},"hexDigit":{"items":{"type":"string"},"type":"array"},"entrySetCallCount":{"format":"int32","type":"integer"},"KEYS":{"format":"int32","type":"integer"},"initialized":{"type":"boolean"},"VALUES":{"format":"int32","type":"integer"},"legacyMap":{"$ref":"#/components/schemas/Map"},"servicesChanged":{"type":"boolean"},"map":{"$ref":"#/components/schemas/ConcurrentHashMap"},"keySet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"table":{"items":{"$ref":"#/components/schemas/Entry"},"type":"array"},"ENTRIES":{"format":"int32","type":"integer"},"info":{"type":"string"},"serviceSet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"debug":{"$ref":"#/components/schemas/Debug"},"previousKey":{"$ref":"#/components/schemas/ServiceKey"},"knownEngines":{"$ref":"#/components/schemas/Map"},"entrySet":{"items":{"type":"object"},"type":"array","uniqueItems":true},"count":{"format":"int32","type":"integer"},"ALIAS_PREFIX":{"type":"string"},"version":{"format":"double","type":"number"},"versionStr":{"type":"string"},"ALIAS_LENGTH":{"format":"int32","type":"integer"},"MAX_ARRAY_SIZE":{"format":"int32","type":"integer"},"defaults":{"$ref":"#/components/schemas/Properties"},"loadFactor":{"format":"float","type":"number"},"legacyChanged":{"type":"boolean"},"name":{"type":"string"},"ALIAS_PREFIX_LOWER":{"type":"string"}},"type":"object"},"Response":{"name":"Response","properties":{"r":{"$ref":"#/components/schemas/Object"}},"type":"object"},"ModuleDescriptor":{"name":"ModuleDescriptor","properties":{"mainClass":{"type":"string"},"exports":{"items":{"type":"object"},"type":"array","uniqueItems":true},"automatic":{"type":"boolean"},"$assertionsDisabled":{"type":"boolean"},"modifiers":{"items":{"type":"object"},"type":"array","uniqueItems":true},"packages":{"items":{"type":"object"},"type":"array","uniqueItems":true},"version":{"$ref":"#/components/schemas/Version"},"provides":{"items":{"type":"object"},"type":"array","uniqueItems":true},"name":{"type":"string"},"opens":{"items":{"type":"object"},"type":"array","uniqueItems":true},"uses":{"items":{"type":"object"},"type":"array","uniqueItems":true},"rawVersionString":{"type":"string"},"open":{"type":"boolean"},"hash":{"format":"int32","type":"integer"},"requires":{"items":{"type":"object"},"type":"array","uniqueItems":true}},"type":"object"},"Bar":{"name":"Bar","properties":{"l1":{"format":"int64","type":"integer"},"l2":{"items":{"format":"int64","type":"integer"},"type":"array"},"foo":{"$ref":"#/components/schemas/Foo"},"f1":{"format":"float","type":"number"},"f2":{"items":{"format":"float","type":"number"},"type":"array"},"d1":{"format":"double","type":"number"},"d2":{"items":{"format":"double","type":"number"},"type":"array"},"b1":{"type":"boolean"},"b2":{"items":{"type":"boolean"},"type":"array"},"ss1":{"items":{"type":"object"},"type":"array","uniqueItems":true},"bar2":{"items":{"$ref":"#/components/schemas/Bar"},"type":"array"},"bar":{"$ref":"#/components/schemas/Bar"},"foo2":{"items":{"$ref":"#/components/schemas/Foo"},"type":"array"},"s1":{"format":"int32","type":"integer"},"s2":{"items":{"format":"int32","type":"integer"},"type":"array"},"m1":{"$ref":"#/components/schemas/Map"},"k1":{"type":"string"},"k2":{"items":{"type":"string"},"type":"array"},"k3":{"items":{"type":"object"},"type":"array"},"i1":{"format":"int32","type":"integer"},"i2":{"items":{"format":"int32","type":"integer"},"type":"array"},"enumnumnum":{"enums":["A","B","C"],"type":"string"},"c1":{"type":"string"},"c2":{"items":{"type":"string"},"type":"array"},"list1":{"items":{"type":"object"},"type":"array"},"list4":{"items":{"type":"object"},"type":"array"},"list3":{"items":{"type":"object"},"type":"array"},"list2":{"items":{"type":"object"},"type":"array"}},"type":"object"},"ClassLoaderValue":{"name":"ClassLoaderValue","properties":{"JLA":{"$ref":"#/components/schemas/JavaLangAccess"}},"type":"object"}}},"info":{"title":"OpenAPI definition"},"openapi":"3.0.1","paths":{"/request-mapping/multi-path/002":{"delete":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"head":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"options":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"patch":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"put":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"trace":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/error":{"delete":{"operationId":"org.springframework.boot.autoconfigure.web.servlet.error.BasicErrorController#error","responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/ResponseEntity"}}},"description":"ok"}}},"get":{"operationId":"org.springframework.boot.autoconfigure.web.servlet.error.BasicErrorController#error","responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/ResponseEntity"}}},"description":"ok"}}},"head":{"operationId":"org.springframework.boot.autoconfigure.web.servlet.error.BasicErrorController#error","responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/ResponseEntity"}}},"description":"ok"}}},"options":{"operationId":"org.springframework.boot.autoconfigure.web.servlet.error.BasicErrorController#error","responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/ResponseEntity"}}},"description":"ok"}}},"patch":{"operationId":"org.springframework.boot.autoconfigure.web.servlet.error.BasicErrorController#error","responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/ResponseEntity"}}},"description":"ok"}}},"post":{"operationId":"org.springframework.boot.autoconfigure.web.servlet.error.BasicErrorController#error","responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/ResponseEntity"}}},"description":"ok"}}},"put":{"operationId":"org.springframework.boot.autoconfigure.web.servlet.error.BasicErrorController#error","responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/ResponseEntity"}}},"description":"ok"}}},"trace":{"operationId":"org.springframework.boot.autoconfigure.web.servlet.error.BasicErrorController#error","responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/ResponseEntity"}}},"description":"ok"}}}},"/request-mapping/multi-path/001":{"delete":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"head":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"options":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"patch":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"put":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"trace":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiPath","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/request-mapping/path/{value1}/{value2}":{"delete":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value2","required":false,"schema":{"type":"string"}},{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value2","required":false,"schema":{"type":"string"}},{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"head":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value2","required":false,"schema":{"type":"string"}},{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"options":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value2","required":false,"schema":{"type":"string"}},{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"patch":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value2","required":false,"schema":{"type":"string"}},{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value2","required":false,"schema":{"type":"string"}},{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"put":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value2","required":false,"schema":{"type":"string"}},{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"trace":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value2","required":false,"schema":{"type":"string"}},{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/swagger-ui.html":{"get":{"operationId":"org.springdoc.webmvc.ui.SwaggerWelcomeWebMvc#redirectToUi","responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/ResponseEntity"}}},"description":"ok"}}}},"/delete-mapping":{"delete":{"operationId":"app.iast.api.springmvc.controller.GreetingController#deleteMapping","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/show-parameters/{f1}":{"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#showParameters","parameters":[{"in":"path","name":"f1","required":true,"schema":{"type":"string"}},{"in":"header","name":"f3","required":true,"schema":{"type":"string"}},{"in":"cookie","name":"f4","required":true,"schema":{"type":"string"}},{"in":"query","name":"f5","required":true,"schema":{"type":"string"}}],"requestBody":{"content":{"application/json":{"schema":{"type":"string"}}},"required":true},"responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/Response"}}},"description":"ok"}}}},"/patch-mapping":{"patch":{"operationId":"app.iast.api.springmvc.controller.GreetingController#patch_mapping","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/request-mapping/RequestBody":{"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingHeader0012","parameters":[{"in":"query","name":"name","required":true,"schema":{"type":"string"}}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Foo"}}},"required":true},"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/v3/api-docs":{"get":{"operationId":"org.springdoc.webmvc.api.OpenApiWebMvcResource#openapiJson","parameters":[{"in":"query","name":"apiDocsUrl","required":true,"schema":{"type":"string"}},{"in":"query","name":"locale","required":true,"schema":{"$ref":"#/components/schemas/Locale"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/v3/api-docs/swagger-config":{"get":{"operationId":"org.springdoc.webmvc.ui.SwaggerConfigResource#openapiJson","responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/Map"}}},"description":"ok"}}}},"/request-mapping/produces":{"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingProducesJson","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/request-mapping/RequestParam":{"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingHeader001","parameters":[{"in":"header","name":"foo","required":true,"schema":{"type":"string"}},{"in":"query","name":"name","required":true,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/request-mapping/params":{"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingParamsA1","parameters":[{"in":"query","name":"action","required":true,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/get-mapping":{"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#getMapping","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/generic":{"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#generic","parameters":[{"in":"query","name":"foo","required":true,"schema":{"$ref":"#/components/schemas/Foo"}},{"in":"query","name":"name","required":true,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/Response"}}},"description":"ok"}}}},"/*":{"delete":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingFallback","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingFallback","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"head":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingFallback","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"options":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingFallback","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"patch":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingFallback","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingFallback","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"put":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingFallback","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"trace":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingFallback","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/request-mapping/multi-method":{"delete":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiMethod","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiMethod","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiMethod","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"put":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingMultiMethod","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/test-openapi-data-types":{"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#testOpenApiDataTypes","parameters":[{"in":"query","name":"v6","required":false,"schema":{"type":"boolean"}},{"in":"query","name":"v7","required":false,"schema":{"type":"string"}},{"in":"query","name":"v8","required":false,"schema":{"format":"int32","type":"integer"}},{"in":"query","name":"v9","required":false,"schema":{"items":{"format":"int32","type":"integer"},"type":"array"}},{"in":"query","name":"v10","required":false,"schema":{"$ref":"#/components/schemas/Foo"}},{"in":"query","name":"v1","required":false,"schema":{"type":"string"}},{"in":"query","name":"v2","required":false,"schema":{"format":"float","type":"number"}},{"in":"query","name":"v3","required":false,"schema":{"format":"double","type":"number"}},{"in":"query","name":"v4","required":false,"schema":{"format":"int32","type":"integer"}},{"in":"query","name":"v5","required":false,"schema":{"format":"int64","type":"integer"}}],"responses":{"200":{"content":{"*/*":{"schema":{"$ref":"#/components/schemas/Foo"}}},"description":"ok"}}}},"/post-mapping":{"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#postMapping","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/request-mapping/headers":{"delete":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingHeader001","parameters":[{"in":"header","name":"foo","required":true,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingHeader001","parameters":[{"in":"header","name":"foo","required":true,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"head":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingHeader001","parameters":[{"in":"header","name":"foo","required":true,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"options":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingHeader001","parameters":[{"in":"header","name":"foo","required":true,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"patch":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingHeader001","parameters":[{"in":"header","name":"foo","required":true,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingHeader001","parameters":[{"in":"header","name":"foo","required":true,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"put":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingHeader001","parameters":[{"in":"header","name":"foo","required":true,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"trace":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingHeader001","parameters":[{"in":"header","name":"foo","required":true,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/request-mapping/consumes":{"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingConsumersXml","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/request-mapping/path/{numericId:\\d+}":{"delete":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariableRegexp","parameters":[{"in":"path","name":"numericId","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariableRegexp","parameters":[{"in":"path","name":"numericId","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"head":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariableRegexp","parameters":[{"in":"path","name":"numericId","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"options":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariableRegexp","parameters":[{"in":"path","name":"numericId","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"patch":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariableRegexp","parameters":[{"in":"path","name":"numericId","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariableRegexp","parameters":[{"in":"path","name":"numericId","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"put":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariableRegexp","parameters":[{"in":"path","name":"numericId","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"trace":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariableRegexp","parameters":[{"in":"path","name":"numericId","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/v3/api-docs.yaml":{"get":{"operationId":"org.springdoc.webmvc.api.OpenApiWebMvcResource#openapiYaml","parameters":[{"in":"query","name":"apiDocsUrl","required":true,"schema":{"type":"string"}},{"in":"query","name":"locale","required":true,"schema":{"$ref":"#/components/schemas/Locale"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/request-mapping/path/{value1}":{"delete":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"head":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"options":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"patch":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"put":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"trace":{"operationId":"app.iast.api.springmvc.controller.GreetingController#requestMappingPathVariable","parameters":[{"in":"path","name":"value1","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/test-spring-mvc-api-collect":{"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#testSpringMVCApiCollect","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/put-mapping":{"put":{"operationId":"app.iast.api.springmvc.controller.GreetingController#putMapping","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}},"/request-mapping":{"delete":{"operationId":"app.iast.api.springmvc.controller.GreetingController#greeting","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"get":{"operationId":"app.iast.api.springmvc.controller.GreetingController#greeting","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"head":{"operationId":"app.iast.api.springmvc.controller.GreetingController#greeting","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"options":{"operationId":"app.iast.api.springmvc.controller.GreetingController#greeting","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"patch":{"operationId":"app.iast.api.springmvc.controller.GreetingController#greeting","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"post":{"operationId":"app.iast.api.springmvc.controller.GreetingController#greeting","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"put":{"operationId":"app.iast.api.springmvc.controller.GreetingController#greeting","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}},"trace":{"operationId":"app.iast.api.springmvc.controller.GreetingController#greeting","responses":{"200":{"content":{"*/*":{"schema":{"type":"string"}}},"description":"ok"}}}}}}}} \ No newline at end of file From f7a9f375422b2d1a1c40be4dcf8e43ef109bf5a4 Mon Sep 17 00:00:00 2001 From: st1020 Date: Mon, 26 Jun 2023 18:33:24 +0800 Subject: [PATCH 010/161] fix: fix heartbeat handler save fail --- .../report/handler/heartbeat_handler.py | 6 ++-- test/apiserver/test_agent_heartbeat.py | 35 ++++++++++++------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/dongtai_protocol/report/handler/heartbeat_handler.py b/dongtai_protocol/report/handler/heartbeat_handler.py index 4118b39f0..89f8ddb6c 100644 --- a/dongtai_protocol/report/handler/heartbeat_handler.py +++ b/dongtai_protocol/report/handler/heartbeat_handler.py @@ -77,8 +77,6 @@ def save_heartbeat(self): default_dict['report_queue'] = self.report_queue default_dict['method_queue'] = self.method_queue default_dict['replay_queue'] = self.replay_queue - IastHeartbeat.objects.update_or_create(agent_id=self.agent_id, - defaults=default_dict) elif self.return_queue == 0: if self.req_count is not None: default_dict['req_count'] = self.req_count @@ -93,8 +91,8 @@ def save_heartbeat(self): default_dict['method_queue'] = self.method_queue default_dict['replay_queue'] = self.replay_queue default_dict['disk'] = self.disk - IastHeartbeat.objects.update_or_create(agent_id=self.agent_id, - defaults=default_dict) + IastHeartbeat.objects.update_or_create(agent_id=self.agent_id, + defaults=default_dict) update_agent_cache(self.agent_id, default_dict) def get_result(self, msg=None): diff --git a/test/apiserver/test_agent_heartbeat.py b/test/apiserver/test_agent_heartbeat.py index 96cde85ab..2a04196b9 100644 --- a/test/apiserver/test_agent_heartbeat.py +++ b/test/apiserver/test_agent_heartbeat.py @@ -1,27 +1,36 @@ from test.apiserver.test_agent_base import AgentTestCase from dongtai_engine.tasks import is_alive -from dongtai_common.models.agent import IastAgent from dongtai_common.models.heartbeat import IastHeartbeat from time import time class ApiHeartBeatTestCase(AgentTestCase): - def test_agent_heart_beat(self): data = { - 'detail': { + "detail": { # 'isCoreInstalled': 1, - 'agentId': 28, - 'disk': '{"rate":71}', - 'memory': '{"total":"875MB","rate":7,"use":"65.872MB"}', - 'performance': - '[{"metricsKey":"cpuUsage","collectDate":"2022-12-14 12:02:05.009","metricsValue":{"cpuUsagePercentage":0.16666666666667052}},{"metricsKey":"memoryUsage","collectDate":"2022-12-14 12:02:05.009","metricsValue":{"init":65011712,"systemMaxLimit":9223372036854771712,"committed":124256256,"memUsagePercentage":9.841756766183035,"max":917504000,"used":90298512}},{"metricsKey":"memoryNoHeapUsage","collectDate":"2022-12-14 12:02:05.009","metricsValue":{"init":4121784320,"systemMaxLimit":9223372036854771712,"committed":3886866432,"memUsagePercentage":94.30057786235646,"max":4121784320,"used":3886866432}},{"metricsKey":"garbageInfo","collectDate":"2022-12-14 12:02:05.009","metricsValue":{"collectionInfoList":[{"collectionTime":18049,"collectionCount":2715,"tenured":false,"collectionName":"PS Scavenge"},{"collectionTime":948,"collectionCount":4,"tenured":true,"collectionName":"PS MarkSweep"}]}},{"metricsKey":"threadInfo","collectDate":"2022-12-14 12:02:05.225","metricsValue":{"dongTaiThreadInfoList":[{"cpuUsage":0,"cpuTime":29161574907,"name":"DongTai-IAST-ServerConfigMonitor","id":14},{"cpuUsage":0,"cpuTime":27679480672,"name":"DongTai-IAST-ConfigMonitor","id":15},{"cpuUsage":0.035035544200697016,"cpuTime":13643355644,"name":"DongTai-IAST-PerformanceMonitor","id":16},{"cpuUsage":0,"cpuTime":111735614230,"name":"DongTai-IAST-EngineMonitor","id":17},{"cpuUsage":0,"cpuTime":61038194468,"name":"DongTai-IAST-HearBeatMonitor","id":18},{"cpuUsage":0,"cpuTime":303498408,"name":"DongTai-IAST-SecondFallbackMonitor","id":19},{"cpuUsage":0,"cpuTime":859911477,"name":"DongTai-IAST-DongTaiThreadMonitor","id":20},{"cpuUsage":0,"cpuTime":26344322815,"name":"DongTai-IAST-HeartBeat","id":64}],"threadCount":51,"dongTaiThreadCount":8,"daemonThreadCount":47,"peakThreadCount":53}}]', + "agentId": 28, + "disk": '{"rate":71}', + "memory": '{"total":"875MB","rate":7,"use":"65.872MB"}', + "performance": '[{"metricsKey":"cpuUsage","collectDate":"2022-12-14 12:02:05.009","metricsValue":{"cpuUsagePercentage":0.16666666666667052}},{"metricsKey":"memoryUsage","collectDate":"2022-12-14 12:02:05.009","metricsValue":{"init":65011712,"systemMaxLimit":9223372036854771712,"committed":124256256,"memUsagePercentage":9.841756766183035,"max":917504000,"used":90298512}},{"metricsKey":"memoryNoHeapUsage","collectDate":"2022-12-14 12:02:05.009","metricsValue":{"init":4121784320,"systemMaxLimit":9223372036854771712,"committed":3886866432,"memUsagePercentage":94.30057786235646,"max":4121784320,"used":3886866432}},{"metricsKey":"garbageInfo","collectDate":"2022-12-14 12:02:05.009","metricsValue":{"collectionInfoList":[{"collectionTime":18049,"collectionCount":2715,"tenured":false,"collectionName":"PS Scavenge"},{"collectionTime":948,"collectionCount":4,"tenured":true,"collectionName":"PS MarkSweep"}]}},{"metricsKey":"threadInfo","collectDate":"2022-12-14 12:02:05.225","metricsValue":{"dongTaiThreadInfoList":[{"cpuUsage":0,"cpuTime":29161574907,"name":"DongTai-IAST-ServerConfigMonitor","id":14},{"cpuUsage":0,"cpuTime":27679480672,"name":"DongTai-IAST-ConfigMonitor","id":15},{"cpuUsage":0.035035544200697016,"cpuTime":13643355644,"name":"DongTai-IAST-PerformanceMonitor","id":16},{"cpuUsage":0,"cpuTime":111735614230,"name":"DongTai-IAST-EngineMonitor","id":17},{"cpuUsage":0,"cpuTime":61038194468,"name":"DongTai-IAST-HearBeatMonitor","id":18},{"cpuUsage":0,"cpuTime":303498408,"name":"DongTai-IAST-SecondFallbackMonitor","id":19},{"cpuUsage":0,"cpuTime":859911477,"name":"DongTai-IAST-DongTaiThreadMonitor","id":20},{"cpuUsage":0,"cpuTime":26344322815,"name":"DongTai-IAST-HeartBeat","id":64}],"threadCount":51,"dongTaiThreadCount":8,"daemonThreadCount":47,"peakThreadCount":53}}]', # 'isCoreRunning': 0, - 'returnQueue': 0, - 'cpu': '{"rate":0}' + "returnQueue": 0, + "cpu": '{"rate":0}', }, - 'type': 1 + "type": 1, } - data['detail']['agentId'] = self.agent_id + data["detail"]["agentId"] = self.agent_id res = self.agent_report(data, agentId=self.agent_id) - assert is_alive(self.agent_id, int(time())) + self.assertTrue(is_alive(self.agent_id, int(time()))) + self.assertEqual( + IastHeartbeat.objects.get(agent_id=self.agent_id).cpu, + data["detail"]["cpu"], + ) + self.assertEqual( + IastHeartbeat.objects.get(agent_id=self.agent_id).memory, + data["detail"]["memory"], + ) + self.assertEqual( + IastHeartbeat.objects.get(agent_id=self.agent_id).disk, + data["detail"]["disk"], + ) From 03fd6f66cbbc12ea9400d41c81902addc9883c6a Mon Sep 17 00:00:00 2001 From: st1020 Date: Mon, 26 Jun 2023 18:46:44 +0800 Subject: [PATCH 011/161] fix: fix heartbeat handler save error --- dongtai_protocol/report/handler/heartbeat_handler.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dongtai_protocol/report/handler/heartbeat_handler.py b/dongtai_protocol/report/handler/heartbeat_handler.py index 89f8ddb6c..2cb454949 100644 --- a/dongtai_protocol/report/handler/heartbeat_handler.py +++ b/dongtai_protocol/report/handler/heartbeat_handler.py @@ -83,6 +83,8 @@ def save_heartbeat(self): default_dict['memory'] = self.memory default_dict['cpu'] = self.cpu default_dict['disk'] = self.disk + IastHeartbeat.objects.update_or_create(agent_id=self.agent_id, + defaults=default_dict) else: default_dict['memory'] = self.memory default_dict['cpu'] = self.cpu @@ -91,8 +93,8 @@ def save_heartbeat(self): default_dict['method_queue'] = self.method_queue default_dict['replay_queue'] = self.replay_queue default_dict['disk'] = self.disk - IastHeartbeat.objects.update_or_create(agent_id=self.agent_id, - defaults=default_dict) + IastHeartbeat.objects.update_or_create(agent_id=self.agent_id, + defaults=default_dict) update_agent_cache(self.agent_id, default_dict) def get_result(self, msg=None): From 23803171364ebcc57454f7c32c10b9fd89d8a193 Mon Sep 17 00:00:00 2001 From: st1020 Date: Mon, 26 Jun 2023 18:58:57 +0800 Subject: [PATCH 012/161] style: fix style error --- dongtai_protocol/report/handler/heartbeat_handler.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/dongtai_protocol/report/handler/heartbeat_handler.py b/dongtai_protocol/report/handler/heartbeat_handler.py index 2cb454949..a974da603 100644 --- a/dongtai_protocol/report/handler/heartbeat_handler.py +++ b/dongtai_protocol/report/handler/heartbeat_handler.py @@ -13,13 +13,11 @@ from dongtai_common.models.vulnerablity import IastVulnerabilityModel from dongtai_common.utils import const from django.utils.translation import gettext_lazy as _ -from dongtai_common.models.server import IastServer from dongtai_protocol.report.handler.report_handler_interface import IReportHandler from dongtai_protocol.report.report_handler_factory import ReportHandler -from django.db.models import (QuerySet, Q, F) -from dongtai_common.models.project import IastProject, VulValidation +from django.db.models import (QuerySet, Q) +from dongtai_common.models.project import VulValidation from dongtai_common.utils.systemsettings import get_vul_validate -from dongtai_common.models.agent import IastAgent from django.core.cache import cache logger = logging.getLogger('dongtai.openapi') @@ -83,7 +81,7 @@ def save_heartbeat(self): default_dict['memory'] = self.memory default_dict['cpu'] = self.cpu default_dict['disk'] = self.disk - IastHeartbeat.objects.update_or_create(agent_id=self.agent_id, + IastHeartbeat.objects.update_or_create(agent_id=self.agent_id, defaults=default_dict) else: default_dict['memory'] = self.memory @@ -93,7 +91,7 @@ def save_heartbeat(self): default_dict['method_queue'] = self.method_queue default_dict['replay_queue'] = self.replay_queue default_dict['disk'] = self.disk - IastHeartbeat.objects.update_or_create(agent_id=self.agent_id, + IastHeartbeat.objects.update_or_create(agent_id=self.agent_id, defaults=default_dict) update_agent_cache(self.agent_id, default_dict) From a9ef79a874a4f36b9932c181a4f699c8390e0431 Mon Sep 17 00:00:00 2001 From: tscuite Date: Tue, 27 Jun 2023 12:00:20 +0800 Subject: [PATCH 013/161] feat: update helm --- .../templates/{ => configmaps}/dongtai-cm.yml | 0 .../helm/templates/data/dongtai-mysql.yaml | 37 ++ .../helm/templates/data/dongtai-redis.yaml | 36 ++ .../deployments/dongtai-logrotate.yaml | 43 ++ .../deployments/dongtai-logtsash.yaml | 76 +++ .../templates/deployments/dongtai-server.yaml | 46 ++ .../templates/deployments/dongtai-web.yaml | 44 ++ .../deployments/dongtai-worker-beat.yaml | 38 ++ .../deployments/dongtai-worker-chain.yaml | 40 ++ .../deployments/dongtai-worker-es.yaml | 38 ++ .../deployments/dongtai-worker-high-freq.yaml | 38 ++ .../deployments/dongtai-worker-other.yaml | 38 ++ .../dongtai-worker-report-only.yaml | 37 ++ .../deployments/dongtai-worker-sca.yaml | 38 ++ .../deployments/dongtai-worker-task.yaml | 35 ++ .../helm/templates/dongtai-date.yml | 75 --- .../helm/templates/dongtai-deploy.yml | 503 ------------------ .../helm/templates/dongtai-svc.yaml | 95 ---- .../dongtai-pv.yml | 0 .../service/dongtai-logstash-svc.yaml | 20 + .../helm/templates/service/dongtai-mysql.yaml | 17 + .../helm/templates/service/dongtai-redis.yaml | 17 + .../templates/service/dongtai-server-svc.yaml | 14 + .../templates/service/dongtai-web-svc.yaml | 22 + 24 files changed, 674 insertions(+), 673 deletions(-) rename deploy/kubernetes/helm/templates/{ => configmaps}/dongtai-cm.yml (100%) create mode 100644 deploy/kubernetes/helm/templates/data/dongtai-mysql.yaml create mode 100644 deploy/kubernetes/helm/templates/data/dongtai-redis.yaml create mode 100644 deploy/kubernetes/helm/templates/deployments/dongtai-logrotate.yaml create mode 100644 deploy/kubernetes/helm/templates/deployments/dongtai-logtsash.yaml create mode 100644 deploy/kubernetes/helm/templates/deployments/dongtai-server.yaml create mode 100644 deploy/kubernetes/helm/templates/deployments/dongtai-web.yaml create mode 100644 deploy/kubernetes/helm/templates/deployments/dongtai-worker-beat.yaml create mode 100644 deploy/kubernetes/helm/templates/deployments/dongtai-worker-chain.yaml create mode 100644 deploy/kubernetes/helm/templates/deployments/dongtai-worker-es.yaml create mode 100644 deploy/kubernetes/helm/templates/deployments/dongtai-worker-high-freq.yaml create mode 100644 deploy/kubernetes/helm/templates/deployments/dongtai-worker-other.yaml create mode 100644 deploy/kubernetes/helm/templates/deployments/dongtai-worker-report-only.yaml create mode 100644 deploy/kubernetes/helm/templates/deployments/dongtai-worker-sca.yaml create mode 100644 deploy/kubernetes/helm/templates/deployments/dongtai-worker-task.yaml delete mode 100644 deploy/kubernetes/helm/templates/dongtai-date.yml delete mode 100644 deploy/kubernetes/helm/templates/dongtai-deploy.yml delete mode 100644 deploy/kubernetes/helm/templates/dongtai-svc.yaml rename deploy/kubernetes/helm/templates/{ => persistentvolumeclaim}/dongtai-pv.yml (100%) create mode 100644 deploy/kubernetes/helm/templates/service/dongtai-logstash-svc.yaml create mode 100644 deploy/kubernetes/helm/templates/service/dongtai-mysql.yaml create mode 100644 deploy/kubernetes/helm/templates/service/dongtai-redis.yaml create mode 100644 deploy/kubernetes/helm/templates/service/dongtai-server-svc.yaml create mode 100644 deploy/kubernetes/helm/templates/service/dongtai-web-svc.yaml diff --git a/deploy/kubernetes/helm/templates/dongtai-cm.yml b/deploy/kubernetes/helm/templates/configmaps/dongtai-cm.yml similarity index 100% rename from deploy/kubernetes/helm/templates/dongtai-cm.yml rename to deploy/kubernetes/helm/templates/configmaps/dongtai-cm.yml diff --git a/deploy/kubernetes/helm/templates/data/dongtai-mysql.yaml b/deploy/kubernetes/helm/templates/data/dongtai-mysql.yaml new file mode 100644 index 000000000..77d571756 --- /dev/null +++ b/deploy/kubernetes/helm/templates/data/dongtai-mysql.yaml @@ -0,0 +1,37 @@ +{{- if not .Values.skipMysql -}} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "dongtai.fullname" . }}-mysql + namespace: {{.Release.Namespace}} +spec: + selector: + matchLabels: + app: {{ template "dongtai.fullname" . }}-mysql + release: {{ .Release.Name }} + {{- include "dongtai.labels" . | nindent 6 }} + strategy: + type: Recreate + template: + metadata: + annotations: + build_number: "{{ template "dongtai.fullname" . }}" + {{- if not .Values.skipistio }}{{ include "dongtai.istiolabels" . | nindent 8 }}{{ end }} + labels: + app: {{ template "dongtai.fullname" . }}-mysql + release: {{ .Release.Name }} + {{- include "dongtai.labels" . | nindent 8 }} + spec: +{{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} + containers: + - image: {{ .Values.images }}/dongtai-mysql:{{ .Values.tag }} + name: mysql-container + imagePullPolicy: Always + ports: + - containerPort: 3306 + name: tcp-mysql +{{- end -}} \ No newline at end of file diff --git a/deploy/kubernetes/helm/templates/data/dongtai-redis.yaml b/deploy/kubernetes/helm/templates/data/dongtai-redis.yaml new file mode 100644 index 000000000..d2bedb486 --- /dev/null +++ b/deploy/kubernetes/helm/templates/data/dongtai-redis.yaml @@ -0,0 +1,36 @@ +{{- if not .Values.skipRedis -}} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ template "dongtai.fullname" . }}-redis + namespace: {{.Release.Namespace}} +spec: + selector: + matchLabels: + app: {{ template "dongtai.fullname" . }}-redis + {{- include "dongtai.labels" . | nindent 6}} + serviceName: dongtai-redis + template: + metadata: + annotations: + build_number: "{{ template "dongtai.fullname" . }}" + {{- if not .Values.skipistio }}{{ include "dongtai.istiolabels" . | nindent 8 }}{{ end }} + labels: + app: {{ template "dongtai.fullname" . }}-redis + release: {{ .Release.Name }} + {{- include "dongtai.labels" . | nindent 8 }} + spec: +{{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} + containers: + - image: {{ .Values.images }}/dongtai-redis:{{ .Values.tag }} + imagePullPolicy: Always + name: {{ template "dongtai.fullname" . }}-redis + ports: + - containerPort: 6379 + name: redis + protocol: TCP +{{- end -}} \ No newline at end of file diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-logrotate.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-logrotate.yaml new file mode 100644 index 000000000..1454fa280 --- /dev/null +++ b/deploy/kubernetes/helm/templates/deployments/dongtai-logrotate.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "dongtai.fullname" . }}-logrotate + namespace: {{.Release.Namespace}} + labels: + app: {{ template "dongtai.fullname" . }}-logrotate + {{- include "dongtai.labels" . | nindent 4 }} +spec: + replicas: {{.Values.replicaCount}} + selector: + matchLabels: + app: {{ template "dongtai.fullname" . }}-logrotate + {{- include "dongtai.labels" . | nindent 6 }} + template: + metadata: + annotations: + build_number: "{{ template "dongtai.fullname" . }}" + {{- if not .Values.skipistio }}{{ include "dongtai.istiolabels" . | nindent 8}}{{ end }} + labels: + app: {{ template "dongtai.fullname" . }}-logrotate + {{- include "dongtai.labels" . | nindent 8 }} + spec: +{{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} + containers: + - image: {{ .Values.images }}/dongtai-logrotate:{{ .Values.tag }} + name: logrotate + securityContext: + runAsUser: 0 + {{ include "deploy.imagePullPolicy" . }} +{{- if .Values.storage.persistentVolumeClaim }} + volumeMounts: + - name: {{ template "dongtai.fullname" . }}-log-path + mountPath: /tmp/logstash + volumes: + - name: {{ template "dongtai.fullname" . }}-log-path + persistentVolumeClaim: + {{ include "deploy.config.persistentVolumeClaim" . }} +{{ end -}} \ No newline at end of file diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-logtsash.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-logtsash.yaml new file mode 100644 index 000000000..ece082964 --- /dev/null +++ b/deploy/kubernetes/helm/templates/deployments/dongtai-logtsash.yaml @@ -0,0 +1,76 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "dongtai.fullname" . }}-logstash + namespace: {{.Release.Namespace}} + labels: + app: {{ template "dongtai.fullname" . }}-logstash + {{- include "dongtai.labels" . | nindent 4 }} +spec: + replicas: {{.Values.replicaCount}} + selector: + matchLabels: + app: {{ template "dongtai.fullname" . }}-logstash + {{- include "dongtai.labels" . | nindent 6 }} + template: + metadata: + annotations: + build_number: "{{ template "dongtai.fullname" . }}" + {{- if not .Values.skipistio }}{{ include "dongtai.istiolabels" . | nindent 8}}{{ end }} + labels: + app: {{ template "dongtai.fullname" . }}-logstash + {{- include "dongtai.labels" . | nindent 8 }} + spec: +{{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} + containers: + - image: {{ .Values.images }}/dongtai-logstash:{{ .Values.tag }} + name: logstash + securityContext: + runAsUser: 0 + {{ include "deploy.imagePullPolicy" . }} + env: + - name: DATABASE + valueFrom: + configMapKeyRef: + key: database + name: logstash-cm + - name: USERNAME + valueFrom: + configMapKeyRef: + key: username + name: logstash-cm + - name: PASSWORD + valueFrom: + configMapKeyRef: + key: password + name: logstash-cm + ports: + - containerPort: 8082 + protocol: TCP + name: agent-http + - containerPort: 8083 + protocol: TCP + name: log-http + resources: + requests: + cpu: 1000m + memory: 2000Mi + limits: + cpu: 2000m + memory: 4000Mi +{{- if .Values.storage.persistentVolumeClaim }} + volumeMounts: + - name: {{ template "dongtai.fullname" . }}-log-path + mountPath: /tmp/logstash + volumes: + - name: {{ template "dongtai.fullname" . }}-log-path + persistentVolumeClaim: + {{ include "deploy.config.persistentVolumeClaim" . }} +{{ end -}} + {{- if .Values.somaxconn }} + {{- include "deploy.initContainers" . | nindent 6 }} + {{- end }} \ No newline at end of file diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-server.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-server.yaml new file mode 100644 index 000000000..ca3e147cf --- /dev/null +++ b/deploy/kubernetes/helm/templates/deployments/dongtai-server.yaml @@ -0,0 +1,46 @@ +--- +# dongtai-server服务 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "dongtai.fullname" . }}-server + namespace: {{.Release.Namespace}} + annotations: + kubesphere.io/description: server + labels: + app: {{ template "dongtai.fullname" . }}-server + {{- include "dongtai.labels" . | nindent 4 }} +spec: + replicas: {{.Values.replicaCount}} + selector: + matchLabels: + app: {{ template "dongtai.fullname" . }}-server + {{- include "dongtai.labels" . | nindent 6 }} + template: + metadata: + annotations: + build_number: "{{ template "dongtai.fullname" . }}" + {{- if not .Values.skipistio }}{{ include "dongtai.istiolabels" . | nindent 8 }}{{ end }} + labels: + app: {{ template "dongtai.fullname" . }}-server + {{- include "dongtai.labels" . | nindent 8 }} + spec: +{{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} + containers: + - name: {{ template "dongtai.fullname" . }}-server-container + image: {{ .Values.images }}/dongtai-server:{{ .Values.tag }} + env: + - name: DONGTAI_CONCURRENCY + value: --processes 4 + {{- range $key, $value := .Values.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- include "deploy.config" . | nindent 10 }} + {{- include "deploy.config.vo" . | nindent 6 }} + {{- if .Values.somaxconn }} + {{- include "deploy.initContainers" . | nindent 6 }} + {{- end }} \ No newline at end of file diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-web.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-web.yaml new file mode 100644 index 000000000..eda74f024 --- /dev/null +++ b/deploy/kubernetes/helm/templates/deployments/dongtai-web.yaml @@ -0,0 +1,44 @@ +--- +# dongtai-web服务 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "dongtai.fullname" . }}-web + namespace: {{.Release.Namespace}} + annotations: + kubesphere.io/description: web + labels: + app: {{ template "dongtai.fullname" . }}-web + {{- include "dongtai.labels" . | nindent 4 }} +spec: + replicas: {{.Values.replicaCount}} + selector: + matchLabels: + app: {{ template "dongtai.fullname" . }}-web + {{- include "dongtai.labels" . | nindent 6 }} + template: + metadata: + annotations: + build_number: "{{ template "dongtai.fullname" . }}" + {{- if not .Values.skipistio }}{{ include "dongtai.istiolabels" . | nindent 8 }}{{ end }} + labels: + app: {{ template "dongtai.fullname" . }}-web + {{- include "dongtai.labels" . | nindent 8 }} + spec: +{{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} + containers: + - name: {{ template "dongtai.fullname" . }}-web-container + image: {{ .Values.images }}/dongtai-web:{{ .Values.tag }} + imagePullPolicy: Always + {{- include "deploy.resources" . | nindent 10 }} + volumeMounts: + - name: configfile + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + volumes: + - name: configfile + configMap: + name: dongtai-web-nginx-conf \ No newline at end of file diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-beat.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-beat.yaml new file mode 100644 index 000000000..650162601 --- /dev/null +++ b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-beat.yaml @@ -0,0 +1,38 @@ +--- +# dongtai-worker-beat服务 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "dongtai.fullname" . }}-worker-beat + namespace: {{.Release.Namespace}} + annotations: + kubesphere.io/description: {{ template "dongtai.fullname" . }}-worker-beat + labels: + app: {{ template "dongtai.fullname" . }}-worker-beat + {{- include "dongtai.labels" . | nindent 4 }} +spec: + replicas: {{.Values.replicaCount}} + selector: + matchLabels: + app: {{ template "dongtai.fullname" . }}-worker-beat + {{- include "dongtai.labels" . | nindent 6 }} + template: + metadata: + labels: + app: {{ template "dongtai.fullname" . }}-worker-beat + {{- include "dongtai.labels" . | nindent 8 }} + spec: +{{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} + containers: + - name: {{ template "dongtai.fullname" . }}-worker-beat-container + image: {{ .Values.images }}/dongtai-server:{{ .Values.tag }} + command: [ "/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh" ] + args: [ "worker-beat" ] + env: + - name: DONGTAI_CONCURRENCY + value: --concurrency=2 + {{- include "deploy.config" . | nindent 10 }} + {{- include "deploy.config.vo" . | nindent 6 }} \ No newline at end of file diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-chain.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-chain.yaml new file mode 100644 index 000000000..9fbe8a419 --- /dev/null +++ b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-chain.yaml @@ -0,0 +1,40 @@ +{{- if .Values.max }} +--- +# dongtai-worker-chain服务 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "dongtai.fullname" . }}-worker-chain + namespace: {{.Release.Namespace}} + annotations: + kubesphere.io/description: {{ template "dongtai.fullname" . }}-worker-chain + labels: + app: {{ template "dongtai.fullname" . }}-worker-chain + {{- include "dongtai.labels" . | nindent 4 }} +spec: + replicas: {{.Values.replicaCount}} + selector: + matchLabels: + app: {{ template "dongtai.fullname" . }}-worker-chain + {{- include "dongtai.labels" . | nindent 6 }} + template: + metadata: + labels: + app: {{ template "dongtai.fullname" . }}-worker-chain + {{- include "dongtai.labels" . | nindent 8 }} + spec: +{{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} + containers: + - name: {{ template "dongtai.fullname" . }}-worker-chain-container + image: {{ .Values.images }}/dongtai-server:{{ .Values.tag }} + command: [ "/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh" ] + args: [ "worker-chain" ] + env: + - name: DONGTAI_CONCURRENCY + value: --concurrency=12 + {{- include "deploy.config" . | nindent 10 }} + {{- include "deploy.config.vo" . | nindent 6 }} +{{- end }} \ No newline at end of file diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-es.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-es.yaml new file mode 100644 index 000000000..fe801fdd0 --- /dev/null +++ b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-es.yaml @@ -0,0 +1,38 @@ +--- +# dongtai-worker-es服务 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "dongtai.fullname" . }}-worker-es + namespace: {{.Release.Namespace}} + annotations: + kubesphere.io/description: {{ template "dongtai.fullname" . }}-worker-es + labels: + app: {{ template "dongtai.fullname" . }}-worker-es + {{- include "dongtai.labels" . | nindent 4 }} +spec: + replicas: {{.Values.replicaCount}} + selector: + matchLabels: + app: {{ template "dongtai.fullname" . }}-worker-es + {{- include "dongtai.labels" . | nindent 6 }} + template: + metadata: + labels: + app: {{ template "dongtai.fullname" . }}-worker-es + {{- include "dongtai.labels" . | nindent 8 }} + spec: +{{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} + containers: + - name: {{ template "dongtai.fullname" . }}-worker-es-container + image: {{ .Values.images }}/dongtai-server:{{ .Values.tag }} + command: [ "/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh" ] + args: [ "worker-es" ] + env: + - name: DONGTAI_CONCURRENCY + value: -P gevent --concurrency=64 + {{- include "deploy.config" . | nindent 10 }} + {{- include "deploy.config.vo" . | nindent 6 }} \ No newline at end of file diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-high-freq.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-high-freq.yaml new file mode 100644 index 000000000..7f90df696 --- /dev/null +++ b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-high-freq.yaml @@ -0,0 +1,38 @@ +--- +# dongtai-worker-high-freq服务 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "dongtai.fullname" . }}-worker-high-freq + namespace: {{.Release.Namespace}} + annotations: + kubesphere.io/description: {{ template "dongtai.fullname" . }}-worker-high-freq + labels: + app: {{ template "dongtai.fullname" . }}-worker-high-freq + {{- include "dongtai.labels" . | nindent 4 }} +spec: + replicas: {{.Values.replicaCount}} + selector: + matchLabels: + app: {{ template "dongtai.fullname" . }}-worker-high-freq + {{- include "dongtai.labels" . | nindent 6 }} + template: + metadata: + labels: + app: {{ template "dongtai.fullname" . }}-worker-high-freq + {{- include "dongtai.labels" . | nindent 8 }} + spec: +{{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} + containers: + - name: {{ template "dongtai.fullname" . }}-worker-high-freq-container + image: {{ .Values.images }}/dongtai-server:{{ .Values.tag }} + command: [ "/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh" ] + args: [ "worker-high-freq" ] + env: + - name: DONGTAI_CONCURRENCY + value: -P gevent --concurrency=121 + {{- include "deploy.config" . | nindent 10 }} + {{- include "deploy.config.vo" . | nindent 6 }} \ No newline at end of file diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-other.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-other.yaml new file mode 100644 index 000000000..b53d744ab --- /dev/null +++ b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-other.yaml @@ -0,0 +1,38 @@ +--- +# dongtai-worker-other服务 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "dongtai.fullname" . }}-worker-other + namespace: {{.Release.Namespace}} + annotations: + kubesphere.io/description: {{ template "dongtai.fullname" . }}-worker-other + labels: + app: {{ template "dongtai.fullname" . }}-worker-other + {{- include "dongtai.labels" . | nindent 4 }} +spec: + replicas: {{.Values.replicaCount}} + selector: + matchLabels: + app: {{ template "dongtai.fullname" . }}-worker-other + {{- include "dongtai.labels" . | nindent 6 }} + template: + metadata: + labels: + app: {{ template "dongtai.fullname" . }}-worker-other + {{- include "dongtai.labels" . | nindent 8 }} + spec: +{{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} + containers: + - name: {{ template "dongtai.fullname" . }}-worker-other-container + image: {{ .Values.images }}/dongtai-server:{{ .Values.tag }} + command: [ "/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh" ] + args: [ "worker-other" ] + env: + - name: DONGTAI_CONCURRENCY + value: --concurrency=10 + {{- include "deploy.config" . | nindent 10 }} + {{- include "deploy.config.vo" . | nindent 6 }} \ No newline at end of file diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-report-only.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-report-only.yaml new file mode 100644 index 000000000..e9c98cf6b --- /dev/null +++ b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-report-only.yaml @@ -0,0 +1,37 @@ +{{- if .Values.max }} +--- +# dongtai-worker-report-only服务 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "dongtai.fullname" . }}-worker-report-only + namespace: {{.Release.Namespace}} + annotations: + kubesphere.io/description: {{ template "dongtai.fullname" . }}-worker-report-only + labels: + app: {{ template "dongtai.fullname" . }}-worker-report-only + {{- include "dongtai.labels" . | nindent 4 }} +spec: + replicas: {{.Values.replicaCount}} + selector: + matchLabels: + app: {{ template "dongtai.fullname" . }}-worker-report-only + {{- include "dongtai.labels" . | nindent 6 }} + template: + metadata: + labels: + app: {{ template "dongtai.fullname" . }}-worker-report-only + {{- include "dongtai.labels" . | nindent 8 }} + spec: +{{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} + containers: + - name: {{ template "dongtai.fullname" . }}-worker-report-only-container + image: {{ .Values.images }}/dongtai-server:{{ .Values.tag }} + command: [ "/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh" ] + args: [ "worker-report-only" ] + {{- include "deploy.configmax" . | nindent 10 }} + {{- include "deploy.config.vo" . | nindent 6 }} +{{- end }} \ No newline at end of file diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-sca.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-sca.yaml new file mode 100644 index 000000000..17bfdb00b --- /dev/null +++ b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-sca.yaml @@ -0,0 +1,38 @@ +--- +# dongtai-worker-sca服务 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "dongtai.fullname" . }}-worker-sca + namespace: {{.Release.Namespace}} + annotations: + kubesphere.io/description: {{ template "dongtai.fullname" . }}-worker-sca + labels: + app: {{ template "dongtai.fullname" . }}-worker-sca + {{- include "dongtai.labels" . | nindent 4 }} +spec: + replicas: {{.Values.replicaCount}} + selector: + matchLabels: + app: {{ template "dongtai.fullname" . }}-worker-sca + {{- include "dongtai.labels" . | nindent 6 }} + template: + metadata: + labels: + app: {{ template "dongtai.fullname" . }}-worker-sca + {{- include "dongtai.labels" . | nindent 8 }} + spec: +{{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} + containers: + - name: {{ template "dongtai.fullname" . }}-worker-sca-container + image: {{ .Values.images }}/dongtai-server:{{ .Values.tag }} + command: [ "/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh" ] + args: [ "worker-sca" ] + env: + - name: DONGTAI_CONCURRENCY + value: -P gevent --concurrency=10 + {{- include "deploy.config" . | nindent 10 }} + {{- include "deploy.config.vo" . | nindent 6 }} \ No newline at end of file diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-task.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-task.yaml new file mode 100644 index 000000000..0669c81d7 --- /dev/null +++ b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-task.yaml @@ -0,0 +1,35 @@ +--- +#dongtai-worker-task服务 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "dongtai.fullname" . }}-worker-task + namespace: {{.Release.Namespace}} + annotations: + kubesphere.io/description: {{ template "dongtai.fullname" . }}-worker-task + labels: + app: {{ template "dongtai.fullname" . }}-worker-task + {{- include "dongtai.labels" . | nindent 4 }} +spec: + replicas: {{.Values.replicaCount}} + selector: + matchLabels: + app: {{ template "dongtai.fullname" . }}-worker-task + {{- include "dongtai.labels" . | nindent 6 }} + template: + metadata: + labels: + app: {{ template "dongtai.fullname" . }}-worker-task + {{- include "dongtai.labels" . | nindent 8 }} + spec: +{{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} + containers: + - name: {{ template "dongtai.fullname" . }}-worker-task-container + image: {{ .Values.images }}/dongtai-server:{{ .Values.tag }} + command: [ "/bin/bash","/opt/dongtai/deploy/docker/entrypoint.sh" ] + args: [ "beat" ] + {{- include "deploy.config" . | nindent 10 }} + {{- include "deploy.config.vo" . | nindent 6 }} \ No newline at end of file diff --git a/deploy/kubernetes/helm/templates/dongtai-date.yml b/deploy/kubernetes/helm/templates/dongtai-date.yml deleted file mode 100644 index 050851b21..000000000 --- a/deploy/kubernetes/helm/templates/dongtai-date.yml +++ /dev/null @@ -1,75 +0,0 @@ -{{- if not .Values.skipRedis -}} ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{ template "dongtai.fullname" . }}-redis - namespace: {{.Release.Namespace}} -spec: - selector: - matchLabels: - app: {{ template "dongtai.fullname" . }}-redis - {{- include "dongtai.labels" . | nindent 6}} - serviceName: dongtai-redis - template: - metadata: - annotations: - build_number: "{{ template "dongtai.fullname" . }}" - {{- if not .Values.skipistio }}{{ include "dongtai.istiolabels" . | nindent 8 }}{{ end }} - labels: - app: {{ template "dongtai.fullname" . }}-redis - release: {{ .Release.Name }} - {{- include "dongtai.labels" . | nindent 8 }} - spec: -{{- if .Values.nodeSelector }} - nodeSelector: -{{ toYaml .Values.nodeSelector | indent 8 }} -{{- end }} - containers: - - image: {{ .Values.images }}/dongtai-redis:{{ .Values.tag }} - imagePullPolicy: Always - name: {{ template "dongtai.fullname" . }}-redis - ports: - - containerPort: 6379 - name: redis - protocol: TCP ---- -{{- end -}} -{{- if not .Values.skipMysql -}} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "dongtai.fullname" . }}-mysql - namespace: {{.Release.Namespace}} -spec: - selector: - matchLabels: - app: {{ template "dongtai.fullname" . }}-mysql - release: {{ .Release.Name }} - {{- include "dongtai.labels" . | nindent 6 }} - strategy: - type: Recreate - template: - metadata: - annotations: - build_number: "{{ template "dongtai.fullname" . }}" - {{- if not .Values.skipistio }}{{ include "dongtai.istiolabels" . | nindent 8 }}{{ end }} - labels: - app: {{ template "dongtai.fullname" . }}-mysql - release: {{ .Release.Name }} - {{- include "dongtai.labels" . | nindent 8 }} - spec: -{{- if .Values.nodeSelector }} - nodeSelector: -{{ toYaml .Values.nodeSelector | indent 8 }} -{{- end }} - containers: - - image: {{ .Values.images }}/dongtai-mysql:{{ .Values.tag }} - name: mysql-container - imagePullPolicy: Always - ports: - - containerPort: 3306 - name: tcp-mysql ---- -{{- end -}} \ No newline at end of file diff --git a/deploy/kubernetes/helm/templates/dongtai-deploy.yml b/deploy/kubernetes/helm/templates/dongtai-deploy.yml deleted file mode 100644 index 44bd549ce..000000000 --- a/deploy/kubernetes/helm/templates/dongtai-deploy.yml +++ /dev/null @@ -1,503 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "dongtai.fullname" . }}-web - namespace: {{.Release.Namespace}} - annotations: - kubesphere.io/description: web - labels: - app: {{ template "dongtai.fullname" . }}-web - {{- include "dongtai.labels" . | nindent 4 }} -spec: - replicas: {{.Values.replicaCount}} - selector: - matchLabels: - app: {{ template "dongtai.fullname" . }}-web - {{- include "dongtai.labels" . | nindent 6 }} - template: - metadata: - annotations: - build_number: "{{ template "dongtai.fullname" . }}" - {{- if not .Values.skipistio }}{{ include "dongtai.istiolabels" . | nindent 8 }}{{ end }} - labels: - app: {{ template "dongtai.fullname" . }}-web - {{- include "dongtai.labels" . | nindent 8 }} - spec: -{{- if .Values.nodeSelector }} - nodeSelector: -{{ toYaml .Values.nodeSelector | indent 8 }} -{{- end }} - containers: - - name: {{ template "dongtai.fullname" . }}-web-container - image: {{ .Values.images }}/dongtai-web:{{ .Values.tag }} - imagePullPolicy: Always - {{- include "deploy.resources" . | nindent 10 }} - volumeMounts: - - name: configfile - mountPath: /etc/nginx/nginx.conf - subPath: nginx.conf - volumes: - - name: configfile - configMap: - name: dongtai-web-nginx-conf ---- -# dongtai-server服务 -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "dongtai.fullname" . }}-server - namespace: {{.Release.Namespace}} - annotations: - kubesphere.io/description: server - labels: - app: {{ template "dongtai.fullname" . }}-server - {{- include "dongtai.labels" . | nindent 4 }} -spec: - replicas: {{.Values.replicaCount}} - selector: - matchLabels: - app: {{ template "dongtai.fullname" . }}-server - {{- include "dongtai.labels" . | nindent 6 }} - template: - metadata: - annotations: - build_number: "{{ template "dongtai.fullname" . }}" - {{- if not .Values.skipistio }}{{ include "dongtai.istiolabels" . | nindent 8 }}{{ end }} - labels: - app: {{ template "dongtai.fullname" . }}-server - {{- include "dongtai.labels" . | nindent 8 }} - spec: -{{- if .Values.nodeSelector }} - nodeSelector: -{{ toYaml .Values.nodeSelector | indent 8 }} -{{- end }} - containers: - - name: {{ template "dongtai.fullname" . }}-server-container - image: {{ .Values.images }}/dongtai-server:{{ .Values.tag }} - env: - - name: DONGTAI_CONCURRENCY - value: --processes 4 - {{- range $key, $value := .Values.env }} - - name: {{ $key }} - value: {{ $value | quote }} - {{- end }} - {{- include "deploy.config" . | nindent 10 }} - {{- include "deploy.config.vo" . | nindent 6 }} - {{- if .Values.somaxconn }} - {{- include "deploy.initContainers" . | nindent 6 }} - {{- end }} ---- - -#dongtai-worker-task服务 -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "dongtai.fullname" . }}-worker-task - namespace: {{.Release.Namespace}} - annotations: - kubesphere.io/description: {{ template "dongtai.fullname" . }}-worker-task - labels: - app: {{ template "dongtai.fullname" . }}-worker-task - {{- include "dongtai.labels" . | nindent 4 }} -spec: - replicas: {{.Values.replicaCount}} - selector: - matchLabels: - app: {{ template "dongtai.fullname" . }}-worker-task - {{- include "dongtai.labels" . | nindent 6 }} - template: - metadata: - labels: - app: {{ template "dongtai.fullname" . }}-worker-task - {{- include "dongtai.labels" . | nindent 8 }} - spec: -{{- if .Values.nodeSelector }} - nodeSelector: -{{ toYaml .Values.nodeSelector | indent 8 }} -{{- end }} - containers: - - name: {{ template "dongtai.fullname" . }}-worker-task-container - image: {{ .Values.images }}/dongtai-server:{{ .Values.tag }} - command: [ "/bin/bash","/opt/dongtai/deploy/docker/entrypoint.sh" ] - args: [ "beat" ] - {{- include "deploy.config" . | nindent 10 }} - {{- include "deploy.config.vo" . | nindent 6 }} ---- -# dongtai-worker-high-freq服务 -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "dongtai.fullname" . }}-worker-high-freq - namespace: {{.Release.Namespace}} - annotations: - kubesphere.io/description: {{ template "dongtai.fullname" . }}-worker-high-freq - labels: - app: {{ template "dongtai.fullname" . }}-worker-high-freq - {{- include "dongtai.labels" . | nindent 4 }} -spec: - replicas: {{.Values.replicaCount}} - selector: - matchLabels: - app: {{ template "dongtai.fullname" . }}-worker-high-freq - {{- include "dongtai.labels" . | nindent 6 }} - template: - metadata: - labels: - app: {{ template "dongtai.fullname" . }}-worker-high-freq - {{- include "dongtai.labels" . | nindent 8 }} - spec: -{{- if .Values.nodeSelector }} - nodeSelector: -{{ toYaml .Values.nodeSelector | indent 8 }} -{{- end }} - containers: - - name: {{ template "dongtai.fullname" . }}-worker-high-freq-container - image: {{ .Values.images }}/dongtai-server:{{ .Values.tag }} - command: [ "/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh" ] - args: [ "worker-high-freq" ] - env: - - name: DONGTAI_CONCURRENCY - value: -P gevent --concurrency=121 - {{- include "deploy.config" . | nindent 10 }} - {{- include "deploy.config.vo" . | nindent 6 }} ---- -# dongtai-worker-beat服务 -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "dongtai.fullname" . }}-worker-beat - namespace: {{.Release.Namespace}} - annotations: - kubesphere.io/description: {{ template "dongtai.fullname" . }}-worker-beat - labels: - app: {{ template "dongtai.fullname" . }}-worker-beat - {{- include "dongtai.labels" . | nindent 4 }} -spec: - replicas: {{.Values.replicaCount}} - selector: - matchLabels: - app: {{ template "dongtai.fullname" . }}-worker-beat - {{- include "dongtai.labels" . | nindent 6 }} - template: - metadata: - labels: - app: {{ template "dongtai.fullname" . }}-worker-beat - {{- include "dongtai.labels" . | nindent 8 }} - spec: -{{- if .Values.nodeSelector }} - nodeSelector: -{{ toYaml .Values.nodeSelector | indent 8 }} -{{- end }} - containers: - - name: {{ template "dongtai.fullname" . }}-worker-beat-container - image: {{ .Values.images }}/dongtai-server:{{ .Values.tag }} - command: [ "/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh" ] - args: [ "worker-beat" ] - env: - - name: DONGTAI_CONCURRENCY - value: --concurrency=2 - {{- include "deploy.config" . | nindent 10 }} - {{- include "deploy.config.vo" . | nindent 6 }} - ---- -# dongtai-worker-sca服务 -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "dongtai.fullname" . }}-worker-sca - namespace: {{.Release.Namespace}} - annotations: - kubesphere.io/description: {{ template "dongtai.fullname" . }}-worker-sca - labels: - app: {{ template "dongtai.fullname" . }}-worker-sca - {{- include "dongtai.labels" . | nindent 4 }} -spec: - replicas: {{.Values.replicaCount}} - selector: - matchLabels: - app: {{ template "dongtai.fullname" . }}-worker-sca - {{- include "dongtai.labels" . | nindent 6 }} - template: - metadata: - labels: - app: {{ template "dongtai.fullname" . }}-worker-sca - {{- include "dongtai.labels" . | nindent 8 }} - spec: -{{- if .Values.nodeSelector }} - nodeSelector: -{{ toYaml .Values.nodeSelector | indent 8 }} -{{- end }} - containers: - - name: {{ template "dongtai.fullname" . }}-worker-sca-container - image: {{ .Values.images }}/dongtai-server:{{ .Values.tag }} - command: [ "/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh" ] - args: [ "worker-sca" ] - env: - - name: DONGTAI_CONCURRENCY - value: -P gevent --concurrency=10 - {{- include "deploy.config" . | nindent 10 }} - {{- include "deploy.config.vo" . | nindent 6 }} ---- -{{- if .Values.max }} -# dongtai-worker-report-only服务 -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "dongtai.fullname" . }}-worker-report-only - namespace: {{.Release.Namespace}} - annotations: - kubesphere.io/description: {{ template "dongtai.fullname" . }}-worker-report-only - labels: - app: {{ template "dongtai.fullname" . }}-worker-report-only - {{- include "dongtai.labels" . | nindent 4 }} -spec: - replicas: {{.Values.replicaCount}} - selector: - matchLabels: - app: {{ template "dongtai.fullname" . }}-worker-report-only - {{- include "dongtai.labels" . | nindent 6 }} - template: - metadata: - labels: - app: {{ template "dongtai.fullname" . }}-worker-report-only - {{- include "dongtai.labels" . | nindent 8 }} - spec: -{{- if .Values.nodeSelector }} - nodeSelector: -{{ toYaml .Values.nodeSelector | indent 8 }} -{{- end }} - containers: - - name: {{ template "dongtai.fullname" . }}-worker-report-only-container - image: {{ .Values.images }}/dongtai-server:{{ .Values.tag }} - command: [ "/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh" ] - args: [ "worker-report-only" ] - {{- include "deploy.configmax" . | nindent 10 }} - {{- include "deploy.config.vo" . | nindent 6 }} ---- -# dongtai-worker-chain服务 -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "dongtai.fullname" . }}-worker-chain - namespace: {{.Release.Namespace}} - annotations: - kubesphere.io/description: {{ template "dongtai.fullname" . }}-worker-chain - labels: - app: {{ template "dongtai.fullname" . }}-worker-chain - {{- include "dongtai.labels" . | nindent 4 }} -spec: - replicas: {{.Values.replicaCount}} - selector: - matchLabels: - app: {{ template "dongtai.fullname" . }}-worker-chain - {{- include "dongtai.labels" . | nindent 6 }} - template: - metadata: - labels: - app: {{ template "dongtai.fullname" . }}-worker-chain - {{- include "dongtai.labels" . | nindent 8 }} - spec: -{{- if .Values.nodeSelector }} - nodeSelector: -{{ toYaml .Values.nodeSelector | indent 8 }} -{{- end }} - containers: - - name: {{ template "dongtai.fullname" . }}-worker-chain-container - image: {{ .Values.images }}/dongtai-server:{{ .Values.tag }} - command: [ "/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh" ] - args: [ "worker-chain" ] - env: - - name: DONGTAI_CONCURRENCY - value: --concurrency=12 - {{- include "deploy.config" . | nindent 10 }} - {{- include "deploy.config.vo" . | nindent 6 }} ---- -{{- end }} - -# dongtai-worker-other服务 -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "dongtai.fullname" . }}-worker-other - namespace: {{.Release.Namespace}} - annotations: - kubesphere.io/description: {{ template "dongtai.fullname" . }}-worker-other - labels: - app: {{ template "dongtai.fullname" . }}-worker-other - {{- include "dongtai.labels" . | nindent 4 }} -spec: - replicas: {{.Values.replicaCount}} - selector: - matchLabels: - app: {{ template "dongtai.fullname" . }}-worker-other - {{- include "dongtai.labels" . | nindent 6 }} - template: - metadata: - labels: - app: {{ template "dongtai.fullname" . }}-worker-other - {{- include "dongtai.labels" . | nindent 8 }} - spec: -{{- if .Values.nodeSelector }} - nodeSelector: -{{ toYaml .Values.nodeSelector | indent 8 }} -{{- end }} - containers: - - name: {{ template "dongtai.fullname" . }}-worker-other-container - image: {{ .Values.images }}/dongtai-server:{{ .Values.tag }} - command: [ "/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh" ] - args: [ "worker-other" ] - env: - - name: DONGTAI_CONCURRENCY - value: --concurrency=10 - {{- include "deploy.config" . | nindent 10 }} - {{- include "deploy.config.vo" . | nindent 6 }} - - ---- -# dongtai-worker-es服务 -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "dongtai.fullname" . }}-worker-es - namespace: {{.Release.Namespace}} - annotations: - kubesphere.io/description: {{ template "dongtai.fullname" . }}-worker-es - labels: - app: {{ template "dongtai.fullname" . }}-worker-es - {{- include "dongtai.labels" . | nindent 4 }} -spec: - replicas: {{.Values.replicaCount}} - selector: - matchLabels: - app: {{ template "dongtai.fullname" . }}-worker-es - {{- include "dongtai.labels" . | nindent 6 }} - template: - metadata: - labels: - app: {{ template "dongtai.fullname" . }}-worker-es - {{- include "dongtai.labels" . | nindent 8 }} - spec: -{{- if .Values.nodeSelector }} - nodeSelector: -{{ toYaml .Values.nodeSelector | indent 8 }} -{{- end }} - containers: - - name: {{ template "dongtai.fullname" . }}-worker-es-container - image: {{ .Values.images }}/dongtai-server:{{ .Values.tag }} - command: [ "/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh" ] - args: [ "worker-es" ] - env: - - name: DONGTAI_CONCURRENCY - value: -P gevent --concurrency=64 - {{- include "deploy.config" . | nindent 10 }} - {{- include "deploy.config.vo" . | nindent 6 }} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "dongtai.fullname" . }}-logstash - namespace: {{.Release.Namespace}} - labels: - app: {{ template "dongtai.fullname" . }}-logstash - {{- include "dongtai.labels" . | nindent 4 }} -spec: - replicas: {{.Values.replicaCount}} - selector: - matchLabels: - app: {{ template "dongtai.fullname" . }}-logstash - {{- include "dongtai.labels" . | nindent 6 }} - template: - metadata: - annotations: - build_number: "{{ template "dongtai.fullname" . }}" - {{- if not .Values.skipistio }}{{ include "dongtai.istiolabels" . | nindent 8}}{{ end }} - labels: - app: {{ template "dongtai.fullname" . }}-logstash - {{- include "dongtai.labels" . | nindent 8 }} - spec: -{{- if .Values.nodeSelector }} - nodeSelector: -{{ toYaml .Values.nodeSelector | indent 8 }} -{{- end }} - containers: - - image: {{ .Values.images }}/dongtai-logrotate:{{ .Values.tag }} - name: logrotate - securityContext: - runAsUser: 0 - {{ include "deploy.imagePullPolicy" . }} - {{- if .Values.storage.persistentVolumeClaim }} - volumeMounts: - - name: {{ template "dongtai.fullname" . }}-log-path - mountPath: /tmp/logstash - {{- end }} - - image: {{ .Values.images }}/dongtai-logstash:{{ .Values.tag }} - name: logstash - securityContext: - runAsUser: 0 - {{ include "deploy.imagePullPolicy" . }} - env: - - name: DATABASE - valueFrom: - configMapKeyRef: - key: database - name: logstash-cm - - name: USERNAME - valueFrom: - configMapKeyRef: - key: username - name: logstash-cm - - name: PASSWORD - valueFrom: - configMapKeyRef: - key: password - name: logstash-cm - ports: - - containerPort: 8082 - protocol: TCP - name: agent-http - - containerPort: 8083 - protocol: TCP - name: log-http - resources: - requests: - cpu: 1000m - memory: 2000Mi - limits: - cpu: 2000m - memory: 4000Mi -# livenessProbe: -# failureThreshold: 1 -# periodSeconds: 5 -# successThreshold: 1 -# tcpSocket: -# port: 9600 -# timeoutSeconds: 1 -# readinessProbe: -# failureThreshold: 3 -# initialDelaySeconds: 30 -# periodSeconds: 5 -# successThreshold: 1 -# tcpSocket: -# port: 9600 -# timeoutSeconds: 1 -# startupProbe: -# failureThreshold: 40 -# periodSeconds: 5 -# successThreshold: 1 -# tcpSocket: -# port: 9600 -# timeoutSeconds: 1 -{{- if .Values.storage.persistentVolumeClaim }} - volumeMounts: - - name: {{ template "dongtai.fullname" . }}-log-path - mountPath: /tmp/logstash - volumes: - - name: {{ template "dongtai.fullname" . }}-log-path - persistentVolumeClaim: - {{ include "deploy.config.persistentVolumeClaim" . }} -{{ end -}} - {{- if .Values.somaxconn }} - {{- include "deploy.initContainers" . | nindent 6 }} - {{- end }} ---- diff --git a/deploy/kubernetes/helm/templates/dongtai-svc.yaml b/deploy/kubernetes/helm/templates/dongtai-svc.yaml deleted file mode 100644 index 3a2cedcf6..000000000 --- a/deploy/kubernetes/helm/templates/dongtai-svc.yaml +++ /dev/null @@ -1,95 +0,0 @@ -{{- if not .Values.skipMysql -}} ---- -apiVersion: v1 -kind: Service -metadata: - name: dongtai-mysql - namespace: {{.Release.Namespace}} - labels: - app: {{ template "dongtai.fullname" . }}-mysql -spec: - selector: - app: {{ template "dongtai.fullname" . }}-mysql - ports: - - protocol: TCP - port: 3306 - targetPort: 3306 ---- -{{- end -}} -{{- if not .Values.skipRedis -}} ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: {{ template "dongtai.fullname" . }}-redis - name: dongtai-redis - namespace: {{.Release.Namespace}} -spec: - ports: - - port: 6379 - protocol: TCP - targetPort: 6379 - selector: - app: {{ template "dongtai.fullname" . }}-redis ---- -{{- end -}} ---- -apiVersion: v1 -kind: Service -metadata: - name: dongtai-server-svc - namespace: {{.Release.Namespace}} -spec: - ports: - - port: 80 - protocol: TCP - targetPort: 8000 - selector: - app: {{ template "dongtai.fullname" . }}-server - type: ClusterIP ---- -{{- if or (eq .Values.accessType "NodePort") (eq .Values.accessType "LoadBalancer") (eq .Values.accessType "ClusterIP") -}} -# Expose dongtai-web svc with {{ .Values.accessType }} ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: {{ template "dongtai.fullname" . }}-web - name: dongtai-web-svc - namespace: {{ .Release.Namespace }} -spec: - ports: - - port: 80 - protocol: TCP - targetPort: 80 - {{- if (eq .Values.accessType "NodePort") }} - nodePort: {{.Values.NodePort}} - {{- end }} - selector: - app: {{ template "dongtai.fullname" . }}-web - type: {{ .Values.accessType }} ---- -{{- end -}} ---- -#dongtai-logstash服务 -apiVersion: v1 -kind: Service -metadata: - name: dongtai-logstash-svc - namespace: {{.Release.Namespace}} -spec: - type: ClusterIP - ports: - - name: agent-http - port: 8082 - targetPort: 8082 - protocol: TCP - - name: log-http - port: 8083 - targetPort: 8083 - protocol: TCP - selector: - app: {{ template "dongtai.fullname" . }}-logstash ---- diff --git a/deploy/kubernetes/helm/templates/dongtai-pv.yml b/deploy/kubernetes/helm/templates/persistentvolumeclaim/dongtai-pv.yml similarity index 100% rename from deploy/kubernetes/helm/templates/dongtai-pv.yml rename to deploy/kubernetes/helm/templates/persistentvolumeclaim/dongtai-pv.yml diff --git a/deploy/kubernetes/helm/templates/service/dongtai-logstash-svc.yaml b/deploy/kubernetes/helm/templates/service/dongtai-logstash-svc.yaml new file mode 100644 index 000000000..106b66656 --- /dev/null +++ b/deploy/kubernetes/helm/templates/service/dongtai-logstash-svc.yaml @@ -0,0 +1,20 @@ +--- +#dongtai-logstash服务 +apiVersion: v1 +kind: Service +metadata: + name: dongtai-logstash-svc + namespace: {{.Release.Namespace}} +spec: + type: ClusterIP + ports: + - name: agent-http + port: 8082 + targetPort: 8082 + protocol: TCP + - name: log-http + port: 8083 + targetPort: 8083 + protocol: TCP + selector: + app: {{ template "dongtai.fullname" . }}-logstash \ No newline at end of file diff --git a/deploy/kubernetes/helm/templates/service/dongtai-mysql.yaml b/deploy/kubernetes/helm/templates/service/dongtai-mysql.yaml new file mode 100644 index 000000000..e510f2c31 --- /dev/null +++ b/deploy/kubernetes/helm/templates/service/dongtai-mysql.yaml @@ -0,0 +1,17 @@ +{{- if not .Values.skipMysql -}} +--- +apiVersion: v1 +kind: Service +metadata: + name: dongtai-mysql + namespace: {{.Release.Namespace}} + labels: + app: {{ template "dongtai.fullname" . }}-mysql +spec: + selector: + app: {{ template "dongtai.fullname" . }}-mysql + ports: + - protocol: TCP + port: 3306 + targetPort: 3306 +{{- end -}} \ No newline at end of file diff --git a/deploy/kubernetes/helm/templates/service/dongtai-redis.yaml b/deploy/kubernetes/helm/templates/service/dongtai-redis.yaml new file mode 100644 index 000000000..df7a8f00f --- /dev/null +++ b/deploy/kubernetes/helm/templates/service/dongtai-redis.yaml @@ -0,0 +1,17 @@ +{{- if not .Values.skipRedis -}} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: {{ template "dongtai.fullname" . }}-redis + name: dongtai-redis + namespace: {{.Release.Namespace}} +spec: + ports: + - port: 6379 + protocol: TCP + targetPort: 6379 + selector: + app: {{ template "dongtai.fullname" . }}-redis +{{- end -}} \ No newline at end of file diff --git a/deploy/kubernetes/helm/templates/service/dongtai-server-svc.yaml b/deploy/kubernetes/helm/templates/service/dongtai-server-svc.yaml new file mode 100644 index 000000000..852ebaa92 --- /dev/null +++ b/deploy/kubernetes/helm/templates/service/dongtai-server-svc.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: dongtai-server-svc + namespace: {{.Release.Namespace}} +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 8000 + selector: + app: {{ template "dongtai.fullname" . }}-server + type: ClusterIP \ No newline at end of file diff --git a/deploy/kubernetes/helm/templates/service/dongtai-web-svc.yaml b/deploy/kubernetes/helm/templates/service/dongtai-web-svc.yaml new file mode 100644 index 000000000..a3e101681 --- /dev/null +++ b/deploy/kubernetes/helm/templates/service/dongtai-web-svc.yaml @@ -0,0 +1,22 @@ +{{- if or (eq .Values.accessType "NodePort") (eq .Values.accessType "LoadBalancer") (eq .Values.accessType "ClusterIP") -}} +# Expose dongtai-web svc with {{ .Values.accessType }} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: {{ template "dongtai.fullname" . }}-web + name: dongtai-web-svc + namespace: {{ .Release.Namespace }} +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + {{- if (eq .Values.accessType "NodePort") }} + nodePort: {{.Values.NodePort}} + {{- end }} + selector: + app: {{ template "dongtai.fullname" . }}-web + type: {{ .Values.accessType }} +{{- end -}} \ No newline at end of file From 108d5b08e64cd382fdf29c9f276e0cd453188e94 Mon Sep 17 00:00:00 2001 From: tscuite Date: Tue, 27 Jun 2023 12:12:02 +0800 Subject: [PATCH 014/161] feat: update helm --- deploy/kubernetes/helm/templates/_helpers.tpl | 18 +++++++++++++++--- deploy/kubernetes/helm/values.yaml | 6 +++++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/deploy/kubernetes/helm/templates/_helpers.tpl b/deploy/kubernetes/helm/templates/_helpers.tpl index ee5c8eb26..43061ffad 100644 --- a/deploy/kubernetes/helm/templates/_helpers.tpl +++ b/deploy/kubernetes/helm/templates/_helpers.tpl @@ -180,18 +180,23 @@ Create the name of the service account to use name = {{.Values.mysql.name}} user = {{.Values.mysql.user}} password = {{.Values.mysql.password}} + [redis] host = {{.Values.redis.host}} port = {{.Values.redis.port}} password = {{.Values.redis.password}} db = {{.Values.redis.db}} + [engine] - url = {{.Values.enginUrl}} + url = http://dongtai-engine:8000 + [apiserver] - url = {{.Values.apiServer}} + url = http://dongtai-server:8000 + [security] csrf_trust_origins = {{.Values.csrfTrustOrigins}} secret_key = {{.Values.secretKey}} + [smtp] server = {{.Values.smtp.server}} user = {{.Values.smtp.user}} @@ -200,24 +205,29 @@ Create the name of the service account to use ssl = {{.Values.smtp.ssl}} cc_addr = {{.Values.smtp.cc_addr}} port = {{.Values.smtp.port}} + [sca] - base_url = https://sca.huoxian.cn/ + base_url = {{.Values.sca.sca_url}} timeout = 5 token = {{.Values.sca.sca_token}} + [task] retryable = true max_retries = 3 async_send = true async_send_delay = 5 + [log_service] host = dongtai-logstash-svc port = 8083 + [common_file_path] tmp_path = /tmp/logstash report_img = report/img report_pdf = report/pdf report_word = report/word report_excel = report/excel + [elastic_search] enable = false host = http://user:passwd@127.0.0.1:9200 @@ -226,10 +236,12 @@ Create the name of the service account to use asset_index = dongtai-iast-alias-dongtai-v1-asset method_pool_index = dongtai-iast-alias-dongtai-v1-method-pool asset_vul_index = dongtai-iast-alias-dongtai-v1-asset-vul + [other] logging_level = {{.Values.logging_level}} cache_preheat = True domain_vul = {{.Values.Dongtai_url}} + dast_token = {{.Values.usb.usb_token}} {{- end -}} {{/* diff --git a/deploy/kubernetes/helm/values.yaml b/deploy/kubernetes/helm/values.yaml index 618556522..b1df9e58e 100644 --- a/deploy/kubernetes/helm/values.yaml +++ b/deploy/kubernetes/helm/values.yaml @@ -40,7 +40,11 @@ redis: db: 0 sca: - sca_token: + sca_url: https://sca.huoxian.cn/ + sca_token: + +usb: + usb_token: 837e62834b8423418c2a2axxxxxxxxxxxxxxx storage: storageClassName: null From a781021346e9ce6d97b2a7728d31fa97f8824b93 Mon Sep 17 00:00:00 2001 From: tscuite Date: Tue, 27 Jun 2023 12:26:45 +0800 Subject: [PATCH 015/161] feat: update helm --- deploy/kubernetes/helm/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/kubernetes/helm/values.yaml b/deploy/kubernetes/helm/values.yaml index b1df9e58e..4a269958c 100644 --- a/deploy/kubernetes/helm/values.yaml +++ b/deploy/kubernetes/helm/values.yaml @@ -14,7 +14,7 @@ memory: 1500Mi accessType: ClusterIP # NodePort, LoadBalancer, ClusterIP NodePort: 30080 logging_level: INFO # DEBUG, INFO -somaxconn: null #If system max net.core.somaxconn (128) . Example: somaxconn: 1024 +somaxconn: null #If system max net.core.somaxconn (128) . Example: somaxconn: 4096 nodeSelector: kubernetes.io/os: linux From 3a1b83f81f2b96bd958a034efc579d8579e14fa2 Mon Sep 17 00:00:00 2001 From: bidaya0 Date: Tue, 27 Jun 2023 15:17:13 +0800 Subject: [PATCH 016/161] fix: remove not reliable api data. --- dongtai_protocol/report/handler/saas_method_pool_handler.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/dongtai_protocol/report/handler/saas_method_pool_handler.py b/dongtai_protocol/report/handler/saas_method_pool_handler.py index ff7fa19ab..1ddba4e80 100644 --- a/dongtai_protocol/report/handler/saas_method_pool_handler.py +++ b/dongtai_protocol/report/handler/saas_method_pool_handler.py @@ -104,8 +104,6 @@ def save(self): :return: """ headers = SaasMethodPoolHandler.parse_headers(self.http_req_header) - #save_project_header(list(headers.keys()), self.agent_id) - add_new_api_route(self.agent, self.http_uri, self.http_method) import base64 params_dict = get_params_dict(base64.b64decode(self.http_req_header), self.http_req_data, From 23fef5c8cff84f0e57e43f6e71962b4e6ec9b766 Mon Sep 17 00:00:00 2001 From: st1020 Date: Wed, 28 Jun 2023 10:26:00 +0800 Subject: [PATCH 017/161] fix: type hint error --- .github/workflows/teststate.yml | 2 +- dongtai_common/engine/vul_engine.py | 6 +- dongtai_common/engine/vul_engine_v2.py | 6 +- dongtai_conf/urls.py | 3 +- dongtai_engine/filters/utils.py | 2 +- dongtai_engine/plugins/data_clean.py | 2 + dongtai_engine/plugins/strategy_headers.py | 2 +- dongtai_engine/tasks.py | 5 +- .../handler/agent_third_service_handler.py | 5 -- .../handler/report_handler_interface.py | 2 +- dongtai_protocol/tests.py | 8 +++ dongtai_protocol/views/agent_config.py | 5 +- dongtai_protocol/views/agent_download.py | 14 ++--- dongtai_web/dongtai_sca/common/dataclass.py | 58 ++++++++++--------- dongtai_web/dongtai_sca/scan/tests.py | 5 +- dongtai_web/dongtai_sca/scan/utils.py | 17 ++---- dongtai_web/serializers/project.py | 9 ++- dongtai_web/threshold/config_setting.py | 6 +- dongtai_web/urls.py | 3 +- dongtai_web/utils.py | 19 +++--- .../views/engine_method_pool_search.py | 4 +- dongtai_web/views/log_download.py | 12 ++-- dongtai_web/views/scas.py | 2 +- test/apiserver/test_agent_base.py | 1 + .../test_agent_register_with_grouptoken.py | 2 +- test/apiserver/test_config.py | 4 +- test/iast/views/test_engine_hook_rule.py | 7 +-- test/iast/views/test_scan_strategy.py | 6 +- test/iast/views/test_sensitive_info_rule.py | 1 + test/iast/views/test_vul_details.py | 1 + test/iast/views/test_vul_summary.py | 3 +- 31 files changed, 112 insertions(+), 110 deletions(-) diff --git a/.github/workflows/teststate.yml b/.github/workflows/teststate.yml index 79e204b48..e15a932c1 100644 --- a/.github/workflows/teststate.yml +++ b/.github/workflows/teststate.yml @@ -124,7 +124,7 @@ jobs: - name: Django Unit Testing run: | - mypy --show-error-codes --ignore-missing-imports --no-incremental --show-error-codes --check-untyped-defs --disable-error-code var-annotated --disable-error-code list-item --disable-error-code attr-defined --disable-error-code arg-type --disable-error-code assignment --disable-error-code misc --disable-error-code union-attr --disable-error-code index --disable-error-code call-overload --disable-error-code dict-item --disable-error-code truthy-function --disable-error-code operator --disable-error-code name-defined . + mypy --show-error-codes --ignore-missing-imports --no-incremental --show-error-codes --check-untyped-defs --disable-error-code var-annotated --disable-error-code list-item --disable-error-code attr-defined --disable-error-code assignment --disable-error-code misc --disable-error-code union-attr --disable-error-code index --disable-error-code call-overload --disable-error-code dict-item --disable-error-code truthy-function --disable-error-code operator --disable-error-code name-defined . Run-Pep8Check: runs-on: ubuntu-latest diff --git a/dongtai_common/engine/vul_engine.py b/dongtai_common/engine/vul_engine.py index 0cda37ffa..570541b98 100644 --- a/dongtai_common/engine/vul_engine.py +++ b/dongtai_common/engine/vul_engine.py @@ -22,11 +22,11 @@ def __init__(self): """ 构造函数，初始化相关数据 """ - self._method_pool = None - self.method_pool_asc = None + self._method_pool = [] + self.method_pool_asc = [] self._vul_method_signature = None self.hit_vul = False - self.vul_stack = None + self.vul_stack = [] self.pool_value = None self.vul_source_signature = None self.graph_data = {'nodes': [], 'edges': []} diff --git a/dongtai_common/engine/vul_engine_v2.py b/dongtai_common/engine/vul_engine_v2.py index ab1f5c70a..ac2809af5 100644 --- a/dongtai_common/engine/vul_engine_v2.py +++ b/dongtai_common/engine/vul_engine_v2.py @@ -18,11 +18,11 @@ def __init__(self): """ 构造函数，初始化相关数据 """ - self._method_pool = None - self.method_pool_asc = None + self._method_pool = [] + self.method_pool_asc = [] self._vul_method_signature = None self.hit_vul = False - self.vul_stack = None + self.vul_stack = [] self.pool_value = None self.vul_source_signature = None self.node_data = {} diff --git a/dongtai_conf/urls.py b/dongtai_conf/urls.py index 439663b5f..65aa5a30f 100644 --- a/dongtai_conf/urls.py +++ b/dongtai_conf/urls.py @@ -18,8 +18,9 @@ import os from dongtai_conf import settings from django.views.decorators.csrf import csrf_exempt +from django.urls import URLPattern, URLResolver -urlpatterns = [ +urlpatterns: list[URLResolver | URLPattern] = [ path('', include('{}.urls'.format(app))) for app in settings.CUSTOM_APPS ] urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) diff --git a/dongtai_engine/filters/utils.py b/dongtai_engine/filters/utils.py index 01eee2121..32bd80b75 100644 --- a/dongtai_engine/filters/utils.py +++ b/dongtai_engine/filters/utils.py @@ -32,4 +32,4 @@ def parse_headers_dict_from_bytes(header_bytes: bytes) -> dict: with SpooledTemporaryFile(max_size=10000) as fp: fp.write(header_bytes) fp.seek(0) - return dict(parse_headers(fp)) + return dict(parse_headers(fp)) # type:ignore diff --git a/dongtai_engine/plugins/data_clean.py b/dongtai_engine/plugins/data_clean.py index d8d1963b0..3df2d97ca 100644 --- a/dongtai_engine/plugins/data_clean.py +++ b/dongtai_engine/plugins/data_clean.py @@ -64,6 +64,8 @@ def data_cleanup(days: int): if not any([latest_id, first_id]): logger.info("no data for clean up") if all([latest_id, first_id]): + assert isinstance(latest_id, int) + assert isinstance(first_id, int) batch_clean(latest_id, first_id, 10000) # qs = MethodPool.objects.filter(pk__lte=latest_id) # qs._raw_delete(qs.db) diff --git a/dongtai_engine/plugins/strategy_headers.py b/dongtai_engine/plugins/strategy_headers.py index 41d3aaa22..e2ef0da95 100644 --- a/dongtai_engine/plugins/strategy_headers.py +++ b/dongtai_engine/plugins/strategy_headers.py @@ -36,7 +36,7 @@ def makefile(self, *args, **kwargs): def parse_response(http_response_str): source = FakeSocket(http_response_str.encode()) - response = HTTPResponse(source) + response = HTTPResponse(source) # type:ignore response.begin() return response diff --git a/dongtai_engine/tasks.py b/dongtai_engine/tasks.py index 7ecaddd94..451693bfd 100644 --- a/dongtai_engine/tasks.py +++ b/dongtai_engine/tasks.py @@ -244,6 +244,7 @@ def search_vul_from_replay_method_pool(method_pool_id): method_pool_model = IastAgentMethodPoolReplay.objects.filter(id=method_pool_id).first() if method_pool_model is None: logger.warn(f'重放数据漏洞检测终止，方法池 {method_pool_id} 不存在') + return strategies = load_sink_strategy(method_pool_model.agent.user, method_pool_model.agent.language) engine = VulEngine() method_pool = json.loads(method_pool_model.method_pool) if method_pool_model else [] @@ -576,7 +577,7 @@ def vul_recheck(): header_raw[index] = f'{_header_name}:{recheck_payload}' break try: - headers = base64.b64encode('\n'.join(header_raw)) + headers = base64.b64encode('\n'.join(header_raw).encode("utf-8")) except Exception as e: logger.warning(f'请求头解析失败，漏洞ID: {vulnerability["id"]}', exc_info=e) elif position == 'COOKIE': @@ -603,7 +604,7 @@ def vul_recheck(): cookie_raw = ';'.join(cookie_raw_items) header_raw[cookie_index] = cookie_raw try: - headers = base64.b64encode('\n'.join(header_raw)) + headers = base64.b64encode('\n'.join(header_raw).encode("utf-8")) except Exception as e: logger.error(f'请求头解析失败，漏洞ID: {vulnerability["id"]}') diff --git a/dongtai_protocol/report/handler/agent_third_service_handler.py b/dongtai_protocol/report/handler/agent_third_service_handler.py index a9d1fbf01..9ee88d69e 100644 --- a/dongtai_protocol/report/handler/agent_third_service_handler.py +++ b/dongtai_protocol/report/handler/agent_third_service_handler.py @@ -8,15 +8,10 @@ from dongtai_protocol.report.handler.report_handler_interface import IReportHandler from dongtai_protocol.report.report_handler_factory import ReportHandler -from dongtai_common.models.api_route import IastApiRoute, IastApiMethod, \ - IastApiResponse, IastApiParameter, \ - IastApiMethodHttpMethodRelation, HttpMethod from dongtai_common.models.agent import IastAgent from dongtai_common.utils import const import logging from django.utils.translation import gettext_lazy as _ -from django.db import transaction -from dongtai_common.models.project import IastProject from dongtai_common.models.agent_thirdservice import IastThirdPartyService logger = logging.getLogger('dongtai.openapi') diff --git a/dongtai_protocol/report/handler/report_handler_interface.py b/dongtai_protocol/report/handler/report_handler_interface.py index cec3705e9..57ab36c1a 100644 --- a/dongtai_protocol/report/handler/report_handler_interface.py +++ b/dongtai_protocol/report/handler/report_handler_interface.py @@ -27,7 +27,7 @@ def __init__(self): self._report = None self._detail = None self._user_id = None - self.agent_id = None + self.agent_id = 0 self.project_name = None self.agent: IastAgent = IastAgent() diff --git a/dongtai_protocol/tests.py b/dongtai_protocol/tests.py index ada6dc4c5..452b33d37 100644 --- a/dongtai_protocol/tests.py +++ b/dongtai_protocol/tests.py @@ -61,6 +61,13 @@ class AgentHeartBeatTestCase(AgentTestCase): def test_agent_replay_queryset(self): self.agent = IastAgent.objects.filter(pk=self.agent_id).first() + assert self.agent is not None + assert self.agent.filepathsimhash is not None + assert self.agent.language is not None + assert self.agent.servicetype is not None + assert self.agent.server is not None + assert self.agent.server.path is not None + assert self.agent.server.hostname is not None project_agents = IastAgent.objects.values_list('id', flat=True).filter( bind_project_id=self.agent.bind_project_id, language=self.agent.language).union( @@ -100,6 +107,7 @@ class AgentSaasMethodPoolParseApiTestCase(AgentTestCase): def test_api_parse(self): mp = MethodPool.objects.filter(pk=500483715).first() + assert mp is not None and mp.req_header is not None mp.req_header headers_bytes = base64.b64decode(mp.req_header) from dongtai_engine.filters.utils import parse_headers_dict_from_bytes diff --git a/dongtai_protocol/views/agent_config.py b/dongtai_protocol/views/agent_config.py index dafc186ae..920867557 100644 --- a/dongtai_protocol/views/agent_config.py +++ b/dongtai_protocol/views/agent_config.py @@ -5,7 +5,6 @@ IastCircuitMetric, TargetType, TargetOperator, - DealType, MetricType, MetricGroup, MetricOperator, @@ -14,13 +13,11 @@ from dongtai_common.endpoint import OpenApiEndPoint, R from dongtai_common.models.agent import IastAgent from dongtai_common.models.agent_config import IastAgentConfig -from django.db.models import Q from drf_spectacular.utils import extend_schema import logging from dongtai_common.utils.systemsettings import get_circuit_break from django.utils.translation import gettext_lazy as _ from result import Ok, Err, Result -from dongtai_common.models.agent_config import MetricGroup logger = logging.getLogger('dongtai.openapi') @@ -124,7 +121,7 @@ def get_function(opt: TargetOperator): def get_filter_by_target(target): targetattr = TargetType(target.target_type).name - opt_function = get_function(TargetType(target.opt)) + opt_function = get_function(TargetOperator(target.opt)) return lambda x: opt_function(x[targetattr], target.value) diff --git a/dongtai_protocol/views/agent_download.py b/dongtai_protocol/views/agent_download.py index 9bc32505c..40cd2272a 100644 --- a/dongtai_protocol/views/agent_download.py +++ b/dongtai_protocol/views/agent_download.py @@ -11,14 +11,14 @@ import logging from django.http import FileResponse -from dongtai_common.endpoint import UserEndPoint, R, OpenApiEndPoint -from drf_spectacular.utils import extend_schema, OpenApiParameter, OpenApiExample +from dongtai_common.endpoint import R, OpenApiEndPoint +from drf_spectacular.utils import extend_schema from rest_framework.authtoken.models import Token from django.utils.translation import gettext_lazy as _ from rest_framework.authentication import SessionAuthentication, TokenAuthentication from dongtai_common.common.utils import DepartmentTokenAuthentication -from dongtai_protocol.api_schema import DongTaiParameter, DongTaiAuth +from dongtai_protocol.api_schema import DongTaiParameter from dongtai_protocol.utils import OssDownloader from dongtai_conf.settings import BUCKET_NAME_BASE_URL, VERSION @@ -124,7 +124,7 @@ def create_config(self, base_url, agent_token, auth_token, project_name, **kwarg agent_file = tarfile.open(user_file) agent_file.extractall( - path=self.target_path, members=lambda memberz: memberz + path=self.target_path ) # trust upstream package until upstream provide file list to validate. names = agent_file.getnames() self.target_source_path = f"{self.target_path}/{names[0]}" @@ -195,7 +195,7 @@ def create_config(self, base_url, agent_token, auth_token, project_name, **kwarg agent_file = tarfile.open(user_file) agent_file.extractall( - path=self.target_path, members=lambda memberz: memberz + path=self.target_path ) # trust upstream package until upstream provide file list to validate. agent_file.close() @@ -286,7 +286,7 @@ def is_tar_file(file): try: agent_file = tarfile.open(file) agent_file.extractall( - path=tmp_path, members=lambda memberz: memberz + path=tmp_path ) # trust upstream package until upstream provide file list to validate. except tarfile.ReadError: return False @@ -309,7 +309,7 @@ def make_download_handler(self, language, user_id): @extend_schema(operation_id="agent download api", tags=[_('Agent Protocol')], - summary=_('Agent download'), + summary=_('Agent download'), # type: ignore parameters=[ DongTaiParameter.OPENAPI_URL, DongTaiParameter.PROJECT_NAME, diff --git a/dongtai_web/dongtai_sca/common/dataclass.py b/dongtai_web/dongtai_sca/common/dataclass.py index 21356765e..a6c5fef0b 100644 --- a/dongtai_web/dongtai_sca/common/dataclass.py +++ b/dongtai_web/dongtai_sca/common/dataclass.py @@ -1,12 +1,9 @@ -from typing import Tuple -from typing import Any -from typing import Optional, Union from dataclasses import dataclass, field from dataclasses_json import dataclass_json, config from datetime import datetime from dateutil.parser import parse -# those Union[Tuple[str], Tuple[()]] = () is not working +# those Union[tuple[str], tuple[()]] = () is not working # Since https://github.com/lidatong/dataclasses-json/pull/409 # Be careful with potentially nullable types when using them temporarily. @@ -21,8 +18,8 @@ class Reference: @dataclass_json @dataclass class VulCodes: - CVE: Union[Tuple[str], Tuple[()]] = () - GHSA: Union[Tuple[str], Tuple[()]] = () + CVE: tuple[str, ...] = () + GHSA: tuple[str, ...] = () @dataclass_json @@ -30,25 +27,30 @@ class VulCodes: class VulInfo: vul_id: str = "" cvss_v3: str = "" - cwe: Union[Tuple[str], Tuple[()]] = () + cwe: tuple[str, ...] = () title: str = "" description: str = "" - references: Tuple[Reference] = () + references: tuple[Reference, ...] = () severity: str = "" - published_time: Optional[datetime] = field( + published_time: datetime | None = field( default=None, - metadata=config(decoder=lambda x: parse(x) if x is not None else None, - encoder=lambda x: datetime.isoformat(x) - if x is not None else None)) - create_time: datetime = field(default=datetime.now(), - metadata=config(decoder=parse, - encoder=datetime.isoformat)) - update_time: datetime = field(default=datetime.now(), - metadata=config(decoder=parse, - encoder=datetime.isoformat)) - change_time: datetime = field(default=datetime.now(), - metadata=config(decoder=parse, - encoder=datetime.isoformat)) + metadata=config( + decoder=lambda x: parse(x) if x is not None else None, + encoder=lambda x: datetime.isoformat(x) if x is not None else None, + ), + ) + create_time: datetime = field( + default=datetime.now(), + metadata=config(decoder=parse, encoder=datetime.isoformat), + ) + update_time: datetime = field( + default=datetime.now(), + metadata=config(decoder=parse, encoder=datetime.isoformat), + ) + change_time: datetime = field( + default=datetime.now(), + metadata=config(decoder=parse, encoder=datetime.isoformat), + ) @dataclass_json @@ -56,16 +58,16 @@ class VulInfo: class Vul: vul_info: VulInfo vul_codes: VulCodes - affected_versions: Union[Tuple[str], Tuple[()]] = () - unaffected_versions: Union[Tuple[str], Tuple[()]] = () + affected_versions: tuple[str, ...] = () + unaffected_versions: tuple[str, ...] = () @dataclass_json @dataclass class PackageVulData: - vuls: Tuple[Vul] = () - affected_versions: Union[Tuple[str], Tuple[()]] = () - unaffected_versions: Union[Tuple[str], Tuple[()]] = () + vuls: tuple[Vul, ...] = () + affected_versions: tuple[str, ...] = () + unaffected_versions: tuple[str, ...] = () @dataclass_json @@ -77,7 +79,7 @@ class PackageInfo: version: str hash: str version_publish_time: str = "" - license: Union[Tuple[str], Tuple[()]] = () + license: tuple[str, ...] = () @dataclass_json @@ -93,4 +95,4 @@ class PackageVulResponse: class PackageResponse: status: int msg: str - data: Tuple[PackageInfo] = tuple() + data: tuple[PackageInfo, ...] = tuple() diff --git a/dongtai_web/dongtai_sca/scan/tests.py b/dongtai_web/dongtai_sca/scan/tests.py index e3addf8e4..d1ab0417a 100644 --- a/dongtai_web/dongtai_sca/scan/tests.py +++ b/dongtai_web/dongtai_sca/scan/tests.py @@ -2,7 +2,7 @@ from .utils import get_nearest_version, get_latest_version from .utils import update_one_sca, new_update_one_sca from test.apiserver.test_agent_base import AgentTestCase -from test import DongTaiTestCase +from typing import cast from .utils import get_package_vul, get_package from django.test import TestCase @@ -169,8 +169,9 @@ def test_update_one_sca_java_result_new(self): agent_id=self.agent_id, signature_value="59fb39a2a8e507785206b42fb8231df0608ff640", ).first() + assert asset is not None # skip until sca data stable - self.assertGreaterEqual(asset.vul_count, 0) + self.assertGreaterEqual(cast(int, asset.vul_count), 0) class SCAScanV2TestCase(AgentTestCase): diff --git a/dongtai_web/dongtai_sca/scan/utils.py b/dongtai_web/dongtai_sca/scan/utils.py index 159e36e7e..3b09d0449 100644 --- a/dongtai_web/dongtai_sca/scan/utils.py +++ b/dongtai_web/dongtai_sca/scan/utils.py @@ -6,7 +6,6 @@ IastVulAssetRelation, IastAssetVulType) from packaging.version import _BaseVersion -from dongtai_common.models.asset_vul import IastAssetVul from collections import defaultdict from hashlib import sha1 from dongtai_conf.settings import SCA_SETUP @@ -23,14 +22,12 @@ from requests.exceptions import RequestException import json from json.decoder import JSONDecodeError -from typing import Optional, Callable, Any, Union +from typing import Optional, Callable from typing import List, Dict, Tuple from requests import Response from dongtai_conf.settings import SCA_BASE_URL, SCA_TIMEOUT, SCA_MAX_RETRY_COUNT from urllib.parse import urljoin from dongtai_common.common.utils import cached_decorator -from dongtai_common.models.profile import IastProfile -from json.decoder import JSONDecodeError from http import HTTPStatus from time import sleep @@ -171,8 +168,7 @@ def data_transfrom_package_vul_v2( def data_transfrom_package_vul_v3( response: Response -) -> Result[Tuple[Union[Tuple[Vul], Tuple[()]], Union[Tuple[str], Tuple[()]], - Union[Tuple[str], Tuple[()]], ], str]: +) -> Result[Tuple[Tuple[Vul, ...], Tuple[str, ...], Tuple[str, ...]], str]: try: #res_data = PackageVulResponse.schema().loads(response.content) res_data = PackageVulResponse.from_json(response.content) @@ -253,8 +249,7 @@ def get_package_vul_v3( ecosystem: str = "", package_version: str = "", package_name: str = "", -) -> Tuple[Union[Tuple[Vul], Tuple[()]], Union[Tuple[str], Tuple[()]], Union[ - Tuple[str], Tuple[()]], ]: +) -> Tuple[Tuple[Vul, ...], Tuple[str, ...], Tuple[str, ...]]: url = urljoin( SCA_BASE_URL, f"/openapi/sca/v3/package/{ecosystem.lower()}/{package_name}/{package_version}/vuls" @@ -945,7 +940,7 @@ def get_license_list(license_list_str: str) -> List[Dict]: }] -def get_license_list_v2(license_list: Tuple[str]) -> List[Dict]: +def get_license_list_v2(license_list: Tuple[str, ...]) -> List[Dict]: filter_none: Callable[[Optional[Dict]], bool] = lambda x: x is not None res = [ LICENSE_DICT[license] for license in license_list @@ -990,8 +985,8 @@ class PackageVulSummary: vul_medium_count: int = 0 vul_low_count: int = 0 vul_info_count: int = 0 - affected_versions: Tuple[str] = () - unaffected_versions: Tuple[str] = () + affected_versions: Tuple[str, ...] = () + unaffected_versions: Tuple[str, ...] = () def get_type_with_cwe(cwe_id: str) -> str: diff --git a/dongtai_web/serializers/project.py b/dongtai_web/serializers/project.py index aba39028c..5eb0ed927 100644 --- a/dongtai_web/serializers/project.py +++ b/dongtai_web/serializers/project.py @@ -15,9 +15,12 @@ from dongtai_common.utils.systemsettings import get_vul_validate from django.db.models import QuerySet from collections import defaultdict +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from django.core.paginator import _SupportsPagination -def get_vul_levels_dict(queryset: QuerySet) -> defaultdict: +def get_vul_levels_dict(queryset: "QuerySet | _SupportsPagination") -> defaultdict: vul_levels = IastVulnerabilityModel.objects.values( 'level__name_value', 'level', 'project_id').filter( project_id__in=list( @@ -30,7 +33,7 @@ def get_vul_levels_dict(queryset: QuerySet) -> defaultdict: return vul_levels_dict -def get_project_language(queryset: QuerySet) -> defaultdict: +def get_project_language(queryset: "QuerySet | _SupportsPagination") -> defaultdict: project_languages = IastAgent.objects.values( 'bind_project_id', 'language').filter(bind_project_id__in=list( queryset.values_list('id', flat=True))).distinct() @@ -40,7 +43,7 @@ def get_project_language(queryset: QuerySet) -> defaultdict: return project_language_dict -def get_agent_count(queryset: QuerySet) -> defaultdict: +def get_agent_count(queryset: "QuerySet | _SupportsPagination") -> defaultdict: agent_counts = IastAgent.objects.values('bind_project_id').filter( bind_project_id__in=list(queryset.values_list( 'id', flat=True))).annotate(agent_count=Count('id')) diff --git a/dongtai_web/threshold/config_setting.py b/dongtai_web/threshold/config_setting.py index d2c421d09..b17267006 100644 --- a/dongtai_web/threshold/config_setting.py +++ b/dongtai_web/threshold/config_setting.py @@ -27,7 +27,6 @@ UNIT_DICT, ) from rest_framework import viewsets -from django.db.models import IntegerChoices from rest_framework import serializers import time @@ -38,7 +37,7 @@ from dongtai_web.serializers.agent_config import AgentConfigSettingSerializer from rest_framework.serializers import ValidationError from rest_framework.utils.serializer_helpers import ReturnDict -from typing import Dict +from typing import Dict, cast from collections import OrderedDict _ResponseSerializer = get_response_serializer(status_msg_keypair=( @@ -190,6 +189,7 @@ def config_update(data, config_id): IastCircuitMetric.objects.filter( circuit_config_id=config_id).delete() obj = IastCircuitConfig.objects.filter(pk=config_id).first() + assert obj is not None for i in data['targets']: create_target(i, obj) @@ -332,7 +332,7 @@ def update(self, request, pk): def reset(self, request, pk): if IastCircuitConfig.objects.filter(pk=pk).exists(): config = IastCircuitConfig.objects.filter(pk=pk, ).first() - mg = MetricGroup(config.metric_group) + mg = MetricGroup(cast(int, config.metric_group)) data = DEFAULT_CIRCUITCONFIG[mg.name] config_update(data, pk) return R.success() diff --git a/dongtai_web/urls.py b/dongtai_web/urls.py index 642d958aa..ba3be9eda 100644 --- a/dongtai_web/urls.py +++ b/dongtai_web/urls.py @@ -160,8 +160,9 @@ from dongtai_web.views.new_project_query import (NewApiRouteSearch, NewProjectVersionList) from dongtai_web.project.recognize_rule import RecognizeRuleViewSet from dongtai_web.enum.hook_rules import HookRuleEnumEndPoint +from django.urls import URLResolver, URLPattern -urlpatterns = [ +urlpatterns: list[URLResolver | URLPattern] = [ path('user/', UserDetailEndPoint.as_view()), path('user/changePassword', UserPassword.as_view()), path('user/login', UserLogin.as_view()), diff --git a/dongtai_web/utils.py b/dongtai_web/utils.py index dcc72c85e..9c905dcbc 100644 --- a/dongtai_web/utils.py +++ b/dongtai_web/utils.py @@ -15,18 +15,18 @@ from rest_framework import serializers from django.utils.translation import gettext_lazy as _ from django.utils.text import format_lazy -from django.utils.translation import get_language from rest_framework.serializers import SerializerMetaclass from functools import reduce from django.db.models import Q import operator import hashlib -from dongtai_common.models.api_route import IastApiRoute, IastApiMethod, IastApiRoute, HttpMethod, IastApiResponse, IastApiMethodHttpMethodRelation +from dongtai_common.models.api_route import HttpMethod, IastApiMethodHttpMethodRelation from dongtai_common.models.agent_method_pool import MethodPool from dongtai_common.models.vulnerablity import IastVulnerabilityModel -from rest_framework.serializers import Serializer from dongtai_conf.settings import OPENAPI -from typing import Optional, List, Dict, Union +from typing import List, Dict, Union, TYPE_CHECKING +if TYPE_CHECKING: + from django.db.models.query import QuerySet, ValuesQuerySet def get_model_field(model, exclude=[], include=[]): @@ -78,7 +78,7 @@ def assemble_query_2(condictions: list, def extend_schema_with_envcheck(querys: list = [], - request_bodys: Optional[Union[List, Dict]] = [], + request_bodys: Union[List, Dict] = [], response_bodys: list = [], response_schema=None, **kwargs): @@ -91,14 +91,13 @@ def myextend_schema(func): 'DOC') or os.getenv( 'DOC', None) == 'TRUE': - from drf_spectacular.utils import extend_schema, OpenApiParameter, OpenApiExample, OpenApiTypes + from drf_spectacular.utils import extend_schema, OpenApiTypes from drf_spectacular.utils import OpenApiResponse parameters = list(filter(lambda x: x, map(_filter_query, querys))) request_examples = list( filter(lambda x: x, map(_filter_request_body, request_bodys))) response_examples = list( - filter(lambda x: x, map(_filter_response_body, - response_bodys))) + filter(lambda x: x, map(_filter_response_body, response_bodys))) examples = request_examples + response_examples if kwargs.get('request', None) and request_examples: kwargs['request'] = {'application/json': OpenApiTypes.OBJECT} @@ -112,7 +111,7 @@ def myextend_schema(func): examples=examples if examples else None, responses={ 200: OpenApiResponse( - description=_('The http status codes are both 200, please use the status and msg field returned by the response data to troubleshoot'), + description=_('The http status codes are both 200, please use the status and msg field returned by the response data to troubleshoot'), # type: ignore response=response_schema)}, **kwargs) funcw = deco(func) @@ -308,5 +307,5 @@ def __call__(self, request): return self.get_response(request) -def dict_transfrom(dic: dict, key: str): +def dict_transfrom(dic: "dict | QuerySet | ValuesQuerySet", key: str): return {i[key]: i for i in dic} diff --git a/dongtai_web/views/engine_method_pool_search.py b/dongtai_web/views/engine_method_pool_search.py index 3cc736b4e..099a81451 100644 --- a/dongtai_web/views/engine_method_pool_search.py +++ b/dongtai_web/views/engine_method_pool_search.py @@ -278,8 +278,8 @@ def post(self, request): filter(lambda _: _['method_pool_id'] == method_pool['id'], vulnerablities)): _ = {} - type_ = list( - filter(lambda x: x is not None, [vulnerablity['strategy__vul_name'], vulnerablity['hook_type__name']])) + type_ = [x for x in [vulnerablity['strategy__vul_name'], + vulnerablity['hook_type__name']] if x is not None] _['vulnerablity_type'] = type_[0] if type_ else '' _['vulnerablity_id'] = vulnerablity['id'] _['vulnerablity_hook_type_id'] = vulnerablity['hook_type_id'] diff --git a/dongtai_web/views/log_download.py b/dongtai_web/views/log_download.py index d6b22acee..c20b62e48 100644 --- a/dongtai_web/views/log_download.py +++ b/dongtai_web/views/log_download.py @@ -2,22 +2,20 @@ import logging import zipfile from dongtai_common.endpoint import UserEndPoint -from io import BytesIO, StringIO +from io import BytesIO from dongtai_common.models.agent import IastAgent from enum import Enum -from django.http import FileResponse, JsonResponse +from django.http import FileResponse from rest_framework import viewsets -import logging from result import Ok, Err, Result from functools import partial from wsgiref.util import FileWrapper -from dongtai_common.utils.user import get_auth_users__by_id from django.http import HttpResponseNotFound from dongtai_common.models.message import IastMessage import threading from django.db.models import Q from django.db import transaction -from dongtai_conf.settings import TMP_COMMON_PATH, AGENT_LOG_DIR +from dongtai_conf.settings import TMP_COMMON_PATH from tempfile import NamedTemporaryFile from dongtai_common.endpoint import R @@ -102,7 +100,7 @@ def get_newest_log_zip(agent_id: int) -> Result: return res -def getzipfilesinmemorty(filenames: list) -> Result[int, BytesIO]: +def getzipfilesinmemorty(filenames: list) -> Result[BytesIO, str]: try: zip_subdir = "logs" s = BytesIO() @@ -118,7 +116,7 @@ def getzipfilesinmemorty(filenames: list) -> Result[int, BytesIO]: return Err('unexcept eror') -def file_newest_N_file_under_path(path: str, N: int) -> Result[int, str]: +def file_newest_N_file_under_path(path: str, N: int) -> Result[list[str], str]: try: files = [ f for f in os.listdir(path) diff --git a/dongtai_web/views/scas.py b/dongtai_web/views/scas.py index 35566b493..75c1c2ad8 100644 --- a/dongtai_web/views/scas.py +++ b/dongtai_web/views/scas.py @@ -394,7 +394,7 @@ def get_vul_list_from_elastic_searchv2(user_id, Q('terms', user_id=user_id_list), Q('terms', is_del=[0]), ] - order_list = ['-signature_value.keyword'] + order_list: list[str | dict] = ['-signature_value.keyword'] if order: order_list.insert(0, {order: {'order': order_type}}) if bind_project_id: diff --git a/test/apiserver/test_agent_base.py b/test/apiserver/test_agent_base.py index 0680df56d..38e4edc4d 100644 --- a/test/apiserver/test_agent_base.py +++ b/test/apiserver/test_agent_base.py @@ -378,6 +378,7 @@ class AgentTestCase(APITestCase): def setUp(self): self.user = User.objects.filter(pk=1).first() + assert self.user is not None self.client.force_authenticate(user=self.user) data = self.register_agent(name='test') self.agent_id = data['id'] diff --git a/test/apiserver/test_agent_register_with_grouptoken.py b/test/apiserver/test_agent_register_with_grouptoken.py index f4b042328..f63746993 100644 --- a/test/apiserver/test_agent_register_with_grouptoken.py +++ b/test/apiserver/test_agent_register_with_grouptoken.py @@ -19,7 +19,7 @@ class AgentNewRegisterGroupTokenTestCase(AgentTestCase): def setUp(self): super().setUp() update_department_data() - self.client.force_authenticate(user=None) + self.client.force_authenticate() self.test_agent_id = [] def test_rep_register(self): diff --git a/test/apiserver/test_config.py b/test/apiserver/test_config.py index 129d48d32..7ef89e192 100644 --- a/test/apiserver/test_config.py +++ b/test/apiserver/test_config.py @@ -5,6 +5,7 @@ get_agent_config_by_scan, ) from dongtai_common.models.user import User +from dongtai_common.models.agent_config import MetricGroup class VulDetailTestCase(APITestCase): @@ -17,10 +18,11 @@ def test_agent_detail_retrieve(self): print(res) def test_target_filter(self): - res = get_agent_config_by_scan(1, 2) + res = get_agent_config_by_scan(1, MetricGroup.SYSTEM) print(res) def test_agent_config_request(self): self.user = User.objects.filter(pk=1).first() + assert self.user is not None self.client.force_authenticate(user=self.user) response = self.client.post('/api/v1/agent/thresholdv2') diff --git a/test/iast/views/test_engine_hook_rule.py b/test/iast/views/test_engine_hook_rule.py index 26530de75..35504f15a 100644 --- a/test/iast/views/test_engine_hook_rule.py +++ b/test/iast/views/test_engine_hook_rule.py @@ -7,13 +7,7 @@ ###################################################################### from rest_framework.test import APITestCase -from django.urls import include, path, reverse from dongtai_common.models.user import User -from dongtai_common.models.agent import IastAgent -from dongtai_common.models.vulnerablity import IastVulnerabilityModel -from dongtai_common.models.hook_type import HookType -import time -import unittest import json from dongtai_common.models.hook_strategy import HookStrategy @@ -22,6 +16,7 @@ class EngineHookRuleTestCase(APITestCase): def setUp(self): self.user = User.objects.filter(pk=1).first() + assert self.user is not None self.client.force_authenticate(user=self.user) def test_create(self): diff --git a/test/iast/views/test_scan_strategy.py b/test/iast/views/test_scan_strategy.py index 0d0662652..82383977a 100644 --- a/test/iast/views/test_scan_strategy.py +++ b/test/iast/views/test_scan_strategy.py @@ -8,8 +8,6 @@ from rest_framework.test import APITestCase -from django.urls import include, path, reverse -from dongtai_web.views.scan_strategys import ScanStrategyViewSet from dongtai_common.models.user import User @@ -18,7 +16,9 @@ def setUp(self): pass def test_create(self): - self.client.force_authenticate(user=User.objects.filter(pk=1).first()) + self.user = User.objects.filter(pk=1).first() + assert self.user is not None + self.client.force_authenticate(user=self.user) response = self.client.get('/api/v1/scan_strategy') print(response.content) assert response.status_code == 200 diff --git a/test/iast/views/test_sensitive_info_rule.py b/test/iast/views/test_sensitive_info_rule.py index 4e5e5d588..00a1226c6 100644 --- a/test/iast/views/test_sensitive_info_rule.py +++ b/test/iast/views/test_sensitive_info_rule.py @@ -14,6 +14,7 @@ class SensitiveInfoRuleTestCase(APITestCase): def setUp(self): self.user = User.objects.filter(pk=1).first() + assert self.user is not None self.client.force_authenticate(user=self.user) def test_sensitive_info_rule_create(self): diff --git a/test/iast/views/test_vul_details.py b/test/iast/views/test_vul_details.py index 57745a251..891f42d0d 100644 --- a/test/iast/views/test_vul_details.py +++ b/test/iast/views/test_vul_details.py @@ -16,6 +16,7 @@ class VulDetailTestCase(APITestCase): def login(self): self.user = User.objects.filter(pk=1).first() + assert self.user is not None self.client.force_authenticate(user=self.user) def setUp(self): diff --git a/test/iast/views/test_vul_summary.py b/test/iast/views/test_vul_summary.py index 12b7d613b..0fb12476d 100644 --- a/test/iast/views/test_vul_summary.py +++ b/test/iast/views/test_vul_summary.py @@ -8,11 +8,9 @@ from rest_framework.test import APITestCase -from django.urls import include, path, reverse from dongtai_common.models.user import User from dongtai_common.models.agent import IastAgent from dongtai_common.models.vulnerablity import IastVulnerabilityModel -from dongtai_common.models.hook_type import HookType import time import unittest @@ -21,6 +19,7 @@ class ScanStrategyTestCase(APITestCase): def setUp(self): self.user = User.objects.filter(pk=1).first() + assert self.user is not None self.client.force_authenticate(user=self.user) agent = IastAgent.objects.create(token='testtoken', version='121231', From 98400d531a9c5dd9912254f89147ee3721469020 Mon Sep 17 00:00:00 2001 From: st1020 Date: Wed, 28 Jun 2023 10:48:36 +0800 Subject: [PATCH 018/161] fix: type hint error --- dongtai_protocol/views/agent_download.py | 17 +++++++---------- dongtai_web/utils.py | 4 +++- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/dongtai_protocol/views/agent_download.py b/dongtai_protocol/views/agent_download.py index 40cd2272a..f10bf3bbe 100644 --- a/dongtai_protocol/views/agent_download.py +++ b/dongtai_protocol/views/agent_download.py @@ -123,9 +123,8 @@ def create_config(self, base_url, agent_token, auth_token, project_name, **kwarg shutil.copyfile(self.original_agent_file, f"{user_file}.bak") agent_file = tarfile.open(user_file) - agent_file.extractall( - path=self.target_path - ) # trust upstream package until upstream provide file list to validate. + agent_file.extractall(path=self.target_path) # nosec + # trust upstream package until upstream provide file list to validate. names = agent_file.getnames() self.target_source_path = f"{self.target_path}/{names[0]}" config_path = "" @@ -194,9 +193,8 @@ def create_config(self, base_url, agent_token, auth_token, project_name, **kwarg shutil.copyfile(self.original_agent_file, f"{user_file}.bak") agent_file = tarfile.open(user_file) - agent_file.extractall( - path=self.target_path - ) # trust upstream package until upstream provide file list to validate. + agent_file.extractall(path=self.target_path) # nosec + # trust upstream package until upstream provide file list to validate. agent_file.close() config_lines = [] @@ -285,9 +283,8 @@ def is_tar_file(file): tmp_path = f"/tmp/.dongtai_agent_test/{time.time_ns()}" try: agent_file = tarfile.open(file) - agent_file.extractall( - path=tmp_path - ) # trust upstream package until upstream provide file list to validate. + agent_file.extractall(path=tmp_path) # nosec + # trust upstream package until upstream provide file list to validate. except tarfile.ReadError: return False except Exception as e: @@ -309,7 +306,7 @@ def make_download_handler(self, language, user_id): @extend_schema(operation_id="agent download api", tags=[_('Agent Protocol')], - summary=_('Agent download'), # type: ignore + summary=_('Agent download'), # type: ignore parameters=[ DongTaiParameter.OPENAPI_URL, DongTaiParameter.PROJECT_NAME, diff --git a/dongtai_web/utils.py b/dongtai_web/utils.py index 9c905dcbc..7367144a8 100644 --- a/dongtai_web/utils.py +++ b/dongtai_web/utils.py @@ -111,7 +111,9 @@ def myextend_schema(func): examples=examples if examples else None, responses={ 200: OpenApiResponse( - description=_('The http status codes are both 200, please use the status and msg field returned by the response data to troubleshoot'), # type: ignore + description=_('The http status codes are both 200, ' + 'please use the status and msg field returned ' + 'by the response data to troubleshoot'), # type: ignore response=response_schema)}, **kwargs) funcw = deco(func) From f48d308881276a3b434904fbe6cc1b450ac0647a Mon Sep 17 00:00:00 2001 From: st1020 Date: Wed, 28 Jun 2023 10:55:15 +0800 Subject: [PATCH 019/161] fix: type hint error --- dongtai_web/utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dongtai_web/utils.py b/dongtai_web/utils.py index 7367144a8..7c1b981dc 100644 --- a/dongtai_web/utils.py +++ b/dongtai_web/utils.py @@ -111,9 +111,7 @@ def myextend_schema(func): examples=examples if examples else None, responses={ 200: OpenApiResponse( - description=_('The http status codes are both 200, ' - 'please use the status and msg field returned ' - 'by the response data to troubleshoot'), # type: ignore + description=_('The http status codes are both 200, please use the status and msg field returned by the response data to troubleshoot'), # type: ignore response=response_schema)}, **kwargs) funcw = deco(func) From 19d6b328930bc2ad8562b019c010b001ca956ba4 Mon Sep 17 00:00:00 2001 From: st1020 Date: Wed, 28 Jun 2023 16:01:35 +0800 Subject: [PATCH 020/161] feat: add django-stubs and change a lot of Model --- dongtai_common/models/agent.py | 64 ++++++---------- dongtai_common/models/agent_config.py | 42 +++++----- dongtai_common/models/agent_method_pool.py | 29 +++---- dongtai_common/models/api_route.py | 14 ++-- dongtai_common/models/asset.py | 69 ++++++++--------- dongtai_common/models/asset_aggr.py | 33 ++++---- dongtai_common/models/asset_vul.py | 55 ++++++-------- dongtai_common/models/asset_vul_v2.py | 36 +++------ dongtai_common/models/assetv2.py | 76 ++++++------------- dongtai_common/models/dast_integration.py | 52 ++++--------- dongtai_common/models/department.py | 2 +- dongtai_common/models/deploy.py | 6 +- dongtai_common/models/document.py | 6 +- .../models/engine_monitoring_indicators.py | 8 +- dongtai_common/models/header_vulnerablity.py | 24 +----- dongtai_common/models/heartbeat.py | 18 ++--- dongtai_common/models/hook_strategy.py | 41 ++++------ dongtai_common/models/hook_type.py | 17 ++--- dongtai_common/models/iast_vul_log.py | 12 ++- dongtai_common/models/message.py | 18 ++--- dongtai_common/models/notify_config.py | 10 +-- dongtai_common/models/profile.py | 2 +- dongtai_common/models/program_language.py | 2 +- dongtai_common/models/project.py | 48 ++++-------- dongtai_common/models/project_report.py | 18 ++--- dongtai_common/models/project_version.py | 19 ++--- dongtai_common/models/recognize_rule.py | 1 - dongtai_common/models/replay_method_pool.py | 28 +++---- dongtai_common/models/replay_queue.py | 1 - dongtai_common/models/res_header.py | 8 +- dongtai_common/models/sensitive_info.py | 22 ++---- dongtai_common/models/server.py | 37 +++++---- dongtai_common/models/strategy.py | 18 ++--- dongtai_common/models/strategy_user.py | 8 +- dongtai_common/models/talent.py | 6 +- dongtai_common/models/tests.py | 2 - dongtai_common/models/url_blacklist.py | 23 +----- dongtai_common/models/user.py | 1 - dongtai_common/models/vul_level.py | 6 +- dongtai_common/models/vul_recheck_payload.py | 3 - dongtai_common/models/vulnerablity.py | 52 ++++++------- dongtai_engine/plugins/data_clean.py | 12 +-- dongtai_engine/plugins/strategy_headers.py | 7 +- .../signals/handlers/vul_handler.py | 8 +- .../report/handler/hardencode_vul_handler.py | 5 +- .../report/handler/narmal_vul_handler.py | 5 +- dongtai_protocol/views/agent_config.py | 2 +- dongtai_protocol/views/hook_profiles.py | 3 +- dongtai_web/dongtai_sca/scan/tests.py | 4 +- dongtai_web/threshold/config_setting.py | 7 +- dongtai_web/views/agents.py | 5 +- .../views/api_route_related_request.py | 9 ++- dongtai_web/views/api_route_search.py | 11 +-- pyproject.toml | 4 + requirements.txt | 62 +++++++-------- 55 files changed, 414 insertions(+), 667 deletions(-) diff --git a/dongtai_common/models/agent.py b/dongtai_common/models/agent.py index 023e59474..24dd5a55f 100644 --- a/dongtai_common/models/agent.py +++ b/dongtai_common/models/agent.py @@ -13,62 +13,47 @@ from dongtai_common.utils.settings import get_managed from dongtai_common.models.project import IastProject from dongtai_common.models.project_version import IastProjectVersion -import json from dongtai_common.models.department import Department from time import time class IastAgent(models.Model): - token = models.CharField(max_length=255, blank=True, null=True) - version = models.CharField(max_length=255, blank=True, null=True) - latest_time = models.IntegerField(blank=True, null=True) + token = models.CharField(max_length=255) + version = models.CharField(max_length=255) + latest_time = models.IntegerField() user = models.ForeignKey(User, models.DO_NOTHING) server = models.ForeignKey( to=IastServer, on_delete=models.DO_NOTHING, related_name='agents', - null=True, related_query_name='agent', verbose_name=_('server'), ) - is_audit = models.IntegerField(blank=True, null=True) - is_running = models.IntegerField(blank=True, null=True) - is_core_running = models.IntegerField(blank=True, null=True) - control = models.IntegerField(blank=True, null=True) - is_control = models.IntegerField(blank=True, null=True) + is_audit = models.IntegerField() + is_running = models.IntegerField() + is_core_running = models.IntegerField() + control = models.IntegerField() + is_control = models.IntegerField() bind_project = models.ForeignKey(IastProject, on_delete=models.CASCADE, - blank=True, - null=True, default=-1) project_version = models.ForeignKey(IastProjectVersion, on_delete=models.CASCADE, - blank=True, - null=True, default=-1) - project_name = models.CharField(max_length=255, blank=True, null=True) - online = models.PositiveSmallIntegerField(blank=True, default=0) - language = models.CharField(max_length=10, blank=True, null=True) - filepathsimhash = models.CharField(max_length=255, - default='', - blank=True, - null=True) - servicetype = models.CharField(max_length=255, - default='', - blank=True, - null=True) - alias = models.CharField(default='', max_length=255, blank=True, null=True) - startup_time = models.IntegerField(default=0, null=False) - register_time = models.IntegerField(default=0, null=False) - actual_running_status = models.IntegerField(default=1, null=False) - except_running_status = models.IntegerField(default=1, null=False) - state_status = models.IntegerField(default=1, null=False) - events = models.JSONField(null=False, default=lambda: ['注册成功']) - department = models.ForeignKey(Department, - models.DO_NOTHING, - blank=True, - null=True) - allow_report = models.IntegerField(default=1, null=False) + project_name = models.CharField(max_length=255) + online = models.PositiveSmallIntegerField(default=0) + language = models.CharField(max_length=10) + filepathsimhash = models.CharField(max_length=255, blank=True, default='') + servicetype = models.CharField(max_length=255, blank=True, default='') + alias = models.CharField(default='', max_length=255, blank=True) + startup_time = models.IntegerField(default=0) + register_time = models.IntegerField(default=0) + actual_running_status = models.IntegerField(default=1) + except_running_status = models.IntegerField(default=1) + state_status = models.IntegerField(default=1) + events = models.JSONField(default=lambda: ['注册成功']) + department = models.ForeignKey(Department, models.DO_NOTHING) + allow_report = models.IntegerField(default=1) class Meta: managed = get_managed() @@ -105,9 +90,8 @@ def update_events_if_need(self): class IastAgentEvent(models.Model): agent = models.ForeignKey(IastAgent, on_delete=models.CASCADE, - related_name='new_events', - null=True) - name = models.CharField(default='', max_length=255, blank=True, null=True) + related_name='new_events') + name = models.CharField(default='', max_length=255) time = models.IntegerField(default=lambda: int(time()), blank=True, null=True) diff --git a/dongtai_common/models/agent_config.py b/dongtai_common/models/agent_config.py index 5f037c6b6..085aef5e0 100644 --- a/dongtai_common/models/agent_config.py +++ b/dongtai_common/models/agent_config.py @@ -10,13 +10,13 @@ class IastAgentConfig(models.Model): user = models.ForeignKey(User, models.DO_NOTHING) details = models.JSONField() - hostname = models.CharField(max_length=255, blank=True, null=True) - ip = models.CharField(max_length=100, blank=True, null=True) - port = models.IntegerField(blank=True, null=True) - cluster_name = models.CharField(max_length=255, blank=True, null=True) - cluster_version = models.CharField(max_length=100, blank=True, null=True) - priority = models.IntegerField(blank=True, null=True) - create_time = models.IntegerField(blank=True, null=True) + hostname = models.CharField(max_length=255, ) + ip = models.CharField(max_length=100, ) + port = models.IntegerField() + cluster_name = models.CharField(max_length=255, ) + cluster_version = models.CharField(max_length=100, ) + priority = models.IntegerField() + create_time = models.IntegerField() class Meta: managed = get_managed() @@ -112,19 +112,19 @@ class ApplicationMetricType(IntegerChoices): class IastCircuitConfig(models.Model): user = models.ForeignKey(User, models.DO_NOTHING) - name = models.CharField(max_length=200, blank=True, null=True) - metric_types = models.CharField(max_length=2000, blank=True, null=True) + name = models.CharField(max_length=200, ) + metric_types = models.CharField(max_length=2000, ) target_types = models.CharField(max_length=2000, blank=True, null=True, db_column='targets') system_type = models.IntegerField(blank=True, null=True) - is_enable = models.IntegerField(blank=True, null=True) - is_deleted = models.IntegerField(default=0, blank=True, null=True) - deal = models.IntegerField(blank=True, null=True) - interval = models.IntegerField(blank=True, null=True, default=30) - metric_group = models.IntegerField(blank=True, null=True) - priority = models.IntegerField(blank=True, null=True) + is_enable = models.IntegerField() + is_deleted = models.IntegerField(default=0, ) + deal = models.IntegerField() + interval = models.IntegerField(default=30) + metric_group = models.IntegerField() + priority = models.IntegerField() create_time = models.IntegerField(blank=True, null=True, default=lambda: int(time())) @@ -140,9 +140,9 @@ class Meta: class IastCircuitTarget(models.Model): circuit_config = models.ForeignKey(IastCircuitConfig, on_delete=models.CASCADE) - target_type = models.IntegerField(blank=True, null=True) - opt = models.IntegerField(blank=True, null=True) - value = models.CharField(max_length=200, blank=True, null=True) + target_type = models.IntegerField() + opt = models.IntegerField() + value = models.CharField(max_length=200, ) class Meta: managed = get_managed() @@ -152,9 +152,9 @@ class Meta: class IastCircuitMetric(models.Model): circuit_config = models.ForeignKey(IastCircuitConfig, on_delete=models.CASCADE) - metric_type = models.IntegerField(blank=True, null=True) - opt = models.IntegerField(blank=True, null=True) - value = models.CharField(max_length=200, blank=True, null=True) + metric_type = models.IntegerField() + opt = models.IntegerField() + value = models.CharField(max_length=200, ) class Meta: managed = get_managed() diff --git a/dongtai_common/models/agent_method_pool.py b/dongtai_common/models/agent_method_pool.py index ac8187d5a..c9cd73027 100644 --- a/dongtai_common/models/agent_method_pool.py +++ b/dongtai_common/models/agent_method_pool.py @@ -7,7 +7,6 @@ from dongtai_conf.settings import METHOD_POOL_INDEX from django_elasticsearch_dsl import Document, fields from django_elasticsearch_dsl.registries import registry -from django_elasticsearch_dsl import Document from django.db import models from django.utils.translation import gettext_lazy as _ @@ -25,31 +24,25 @@ class MethodPool(models.Model): blank=True, null=True, db_constraint=False) - url = models.CharField(max_length=2000, blank=True, null=True) - uri = models.CharField(max_length=2000, blank=True, null=True) - http_method = models.CharField(max_length=10, blank=True, default='') - http_scheme = models.CharField(max_length=20, blank=True, null=True) - http_protocol = models.CharField(max_length=255, blank=True, null=True) + url = models.CharField(max_length=2000) + uri = models.CharField(max_length=2000) + http_method = models.CharField(max_length=10, default='') + http_scheme = models.CharField(max_length=20) + http_protocol = models.CharField(max_length=255) req_header = models.CharField(max_length=2000, blank=True, null=True) req_params = models.CharField(max_length=2000, blank=True, null=True) req_data = models.CharField(max_length=4000, blank=True, null=True) res_header = models.CharField(max_length=1000, blank=True, null=True) res_body = models.TextField(blank=True, null=True) - req_header_fs = models.TextField(blank=True, - null=True, - db_column='req_header_for_search') + req_header_fs = models.TextField(db_column='req_header_for_search') context_path = models.CharField(max_length=255, blank=True, null=True) - method_pool = models.TextField(blank=True, - null=True) # This field type is a guess. + method_pool = models.TextField() # This field type is a guess. pool_sign = models.CharField(unique=True, - max_length=40, - blank=True, - null=True) # This field type is a guess. - clent_ip = models.CharField(max_length=255, blank=True, null=True) - create_time = models.IntegerField(blank=True, null=True) - update_time = models.IntegerField(blank=True, null=True) + max_length=40) # This field type is a guess. + clent_ip = models.CharField(max_length=255) + create_time = models.IntegerField() + update_time = models.IntegerField() uri_sha1 = models.CharField(max_length=40, - blank=True, default='', db_index=True) sinks = models.ManyToManyField( diff --git a/dongtai_common/models/api_route.py b/dongtai_common/models/api_route.py index 037183589..e21df5ebb 100644 --- a/dongtai_common/models/api_route.py +++ b/dongtai_common/models/api_route.py @@ -14,7 +14,7 @@ class HttpMethod(models.Model): - method = models.CharField(max_length=100, blank=True) + method = models.CharField(max_length=100) class Meta: managed = get_managed() @@ -22,9 +22,9 @@ class Meta: class IastApiMethod(models.Model): - method = models.CharField(max_length=100, blank=True) + method = models.CharField(max_length=100) http_method = models.ManyToManyField( - HttpMethod, blank=True, through='IastApiMethodHttpMethodRelation') + HttpMethod, through='IastApiMethodHttpMethodRelation') class Meta: managed = get_managed() @@ -76,13 +76,9 @@ class IastApiRoute(models.Model): choices=FromWhereChoices.choices) project = models.ForeignKey(IastProject, on_delete=models.CASCADE, - blank=True, - null=True, default=-1) project_version = models.ForeignKey(IastProjectVersion, on_delete=models.CASCADE, - blank=True, - null=True, default=-1) is_cover = models.IntegerField(default=0) @@ -93,7 +89,7 @@ class Meta: class IastApiParameter(models.Model): - name = models.CharField(max_length=100, blank=True) + name = models.CharField(max_length=100) parameter_type = models.CharField(max_length=100, blank=True, default='', @@ -112,7 +108,7 @@ class Meta: class IastApiResponse(models.Model): - return_type = models.CharField(max_length=100, blank=True) + return_type = models.CharField(max_length=100) route = models.ForeignKey(IastApiRoute, on_delete=models.CASCADE, db_constraint=False, diff --git a/dongtai_common/models/asset.py b/dongtai_common/models/asset.py index 09174ab55..7759147a4 100644 --- a/dongtai_common/models/asset.py +++ b/dongtai_common/models/asset.py @@ -10,10 +10,7 @@ from django_elasticsearch_dsl.search import Search from dongtai_conf.settings import ASSET_INDEX from django_elasticsearch_dsl import Document, fields -from django.db.models.fields.related import ForeignKey -from dongtai_web.utils import get_model_field from django_elasticsearch_dsl.registries import registry -from django_elasticsearch_dsl import Document from django.db import models from django.utils.translation import gettext_lazy as _ @@ -29,53 +26,49 @@ class Asset(models.Model): id = models.BigAutoField(primary_key=True) - package_name = models.CharField(max_length=255, blank=True, null=True) - package_path = models.CharField(max_length=255, blank=True, null=True) - signature_algorithm = models.CharField(max_length=255, blank=True, null=True) - signature_value = models.CharField(max_length=255, blank=True, null=True) - dt = models.IntegerField(blank=True, null=True) - version = models.CharField(max_length=255, blank=True, null=True) - safe_version = models.CharField(max_length=255, blank=True, null=False, default='') - last_version = models.CharField(max_length=255, blank=True, null=False, default='') - level = models.ForeignKey(IastVulLevel, models.DO_NOTHING, blank=True, null=True, default=4) - vul_count = models.IntegerField(blank=True, null=True) - vul_critical_count = models.IntegerField(default=0, blank=True, null=False) - vul_high_count = models.IntegerField(default=0, blank=True, null=False) - vul_medium_count = models.IntegerField(default=0, blank=True, null=False) - vul_low_count = models.IntegerField(default=0, blank=True, null=False) - vul_info_count = models.IntegerField(default=0, blank=True, null=False) + package_name = models.CharField(max_length=255) + package_path = models.CharField(max_length=255) + signature_algorithm = models.CharField(max_length=255) + signature_value = models.CharField(max_length=255) + dt = models.IntegerField() + version = models.CharField(max_length=255, blank=True) + safe_version = models.CharField(max_length=255, blank=True, default='') + last_version = models.CharField(max_length=255, blank=True, default='') + level = models.ForeignKey(IastVulLevel, models.DO_NOTHING, default=4) + vul_count = models.IntegerField() + vul_critical_count = models.IntegerField(default=0) + vul_high_count = models.IntegerField(default=0) + vul_medium_count = models.IntegerField(default=0) + vul_low_count = models.IntegerField(default=0) + vul_info_count = models.IntegerField(default=0) agent = models.ForeignKey( to=IastAgent, on_delete=models.CASCADE, related_name='assets', related_query_name='asset', verbose_name=_('agent'), - blank=True, - null=True, default=-1 ) - project = models.ForeignKey(IastProject, on_delete=models.CASCADE, blank=True, null=False, default=-1) - project_version = models.ForeignKey(IastProjectVersion, on_delete=models.CASCADE, blank=True, null=False, + project = models.ForeignKey(IastProject, on_delete=models.CASCADE, default=-1) + project_version = models.ForeignKey(IastProjectVersion, on_delete=models.CASCADE, default=-1) - user = models.ForeignKey(User, models.DO_NOTHING, null=False, default=-1) - project_name = models.CharField(max_length=255, blank=True, null=False, default='') - language = models.CharField(max_length=32, blank=True, null=False, default='') - license = models.CharField(max_length=64, blank=True, null=False, default='') - dependency_level = models.IntegerField(null=False, default=0) - parent_dependency_id = models.IntegerField(blank=True, null=False, default=0) - is_del = models.SmallIntegerField(blank=True, null=False, default=0) + user = models.ForeignKey(User, models.DO_NOTHING, default=-1) + project_name = models.CharField(max_length=255, default='') + language = models.CharField(max_length=32, default='') + license = models.CharField(max_length=64, default='') + dependency_level = models.IntegerField(default=0) + parent_dependency_id = models.IntegerField(default=0) + is_del = models.SmallIntegerField(default=0) # 部门id - department = models.ForeignKey(Department, models.DO_NOTHING, blank=True, null=True, default=-1) + department = models.ForeignKey(Department, models.DO_NOTHING, default=-1) # 租户id - talent = models.ForeignKey(Talent, models.DO_NOTHING, blank=True, null=True, default=-1) - safe_version_list = models.JSONField(blank=True, null=True, default=list) - nearest_safe_version = models.JSONField(blank=True, - null=True, - default=str) - latest_safe_version = models.JSONField(blank=True, null=True, default=str) - license_list = models.JSONField(blank=True, null=True, default=list) - highest_license = models.JSONField(blank=True, null=True, default=dict) + talent = models.ForeignKey(Talent, models.DO_NOTHING, default=-1) + safe_version_list = models.JSONField(default=list) + nearest_safe_version = models.JSONField(default=str) + latest_safe_version = models.JSONField(default=str) + license_list = models.JSONField(default=list) + highest_license = models.JSONField(default=dict) class Meta: managed = get_managed() diff --git a/dongtai_common/models/asset_aggr.py b/dongtai_common/models/asset_aggr.py index c5acc58e5..aa0c6b6ab 100644 --- a/dongtai_common/models/asset_aggr.py +++ b/dongtai_common/models/asset_aggr.py @@ -7,29 +7,28 @@ from dongtai_conf.settings import ASSET_AGGR_INDEX from django_elasticsearch_dsl import Document, fields from django_elasticsearch_dsl.registries import registry -from django_elasticsearch_dsl import Document from django.db import models from dongtai_common.models.vul_level import IastVulLevel from dongtai_common.utils.settings import get_managed class AssetAggr(models.Model): - package_name = models.CharField(max_length=255, blank=True, null=True) - signature_value = models.CharField(max_length=255, blank=True, null=True) - version = models.CharField(max_length=255, blank=True, null=True) - safe_version = models.CharField(max_length=255, blank=True, null=False, default='') - last_version = models.CharField(max_length=255, blank=True, null=False, default='') - level = models.ForeignKey(IastVulLevel, models.DO_NOTHING, blank=True, null=True) - vul_count = models.IntegerField(blank=True, null=True) - vul_critical_count = models.IntegerField(default=0, blank=True, null=False) - vul_high_count = models.IntegerField(default=0, blank=True, null=False) - vul_medium_count = models.IntegerField(default=0, blank=True, null=False) - vul_low_count = models.IntegerField(default=0, blank=True, null=False) - vul_info_count = models.IntegerField(default=0, blank=True, null=False) - project_count = models.IntegerField(blank=True, null=False, default=0) - language = models.CharField(max_length=32, blank=True, null=False, default='') - license = models.CharField(max_length=64, blank=True, null=False, default='') - is_del = models.SmallIntegerField(blank=True, null=False, default=0) + package_name = models.CharField(max_length=255) + signature_value = models.CharField(max_length=255, blank=True) + version = models.CharField(max_length=255) + safe_version = models.CharField(max_length=255, blank=True, default='') + last_version = models.CharField(max_length=255, blank=True, default='') + level = models.ForeignKey(IastVulLevel, models.DO_NOTHING) + vul_count = models.IntegerField() + vul_critical_count = models.IntegerField(default=0) + vul_high_count = models.IntegerField(default=0) + vul_medium_count = models.IntegerField(default=0) + vul_low_count = models.IntegerField(default=0) + vul_info_count = models.IntegerField(default=0) + project_count = models.IntegerField(blank=True) + language = models.CharField(max_length=32, default='') + license = models.CharField(max_length=64, blank=True, default='') + is_del = models.SmallIntegerField(default=0) class Meta: managed = get_managed() diff --git a/dongtai_common/models/asset_vul.py b/dongtai_common/models/asset_vul.py index e674a4b95..1b912fc3e 100644 --- a/dongtai_common/models/asset_vul.py +++ b/dongtai_common/models/asset_vul.py @@ -13,35 +13,32 @@ class IastAssetVul(models.Model): - vul_name = models.CharField(max_length=255, blank=True, null=True) - vul_detail = models.TextField(blank=True, null=True) + vul_name = models.CharField(max_length=255) + vul_detail = models.TextField() # 漏洞类型等级 - level = models.ForeignKey(IastVulLevel, - models.DO_NOTHING, - blank=True, - null=True) + level = models.ForeignKey(IastVulLevel, models.DO_NOTHING) license = models.CharField(max_length=50, blank=True, null=True) # 开源许可证 风险等级 # 1 高 2中 3低 0无风险 license_level = models.SmallIntegerField(blank=True, null=True) - aql = models.CharField(max_length=100, blank=True, null=True) - package_name = models.CharField(max_length=100, blank=True, null=True) + aql = models.CharField(max_length=100) + package_name = models.CharField(max_length=100) package_hash = models.CharField(max_length=100, blank=True, null=True) - package_version = models.CharField(max_length=50, blank=True, null=True) + package_version = models.CharField(max_length=50) package_safe_version = models.CharField(max_length=50, blank=True, null=True) package_latest_version = models.CharField(max_length=50, blank=True, null=True) - package_language = models.CharField(max_length=10, blank=True, null=True) - vul_cve_nums = models.JSONField(blank=True, null=True) - vul_serial = models.CharField(max_length=100, blank=True, null=True) # 漏洞编号 CWE|CVE等数据 - have_article = models.SmallIntegerField(blank=True, null=True) - have_poc = models.SmallIntegerField(blank=True, null=True) + package_language = models.CharField(max_length=10) + vul_cve_nums = models.JSONField() + vul_serial = models.CharField(max_length=100) # 漏洞编号 CWE|CVE等数据 + have_article = models.SmallIntegerField() + have_poc = models.SmallIntegerField() cve_code = models.CharField(max_length=64, blank=True, null=True) sid = models.CharField(max_length=64, blank=True, null=True) cve_id = models.IntegerField(blank=True, null=True) vul_publish_time = models.DateTimeField(blank=True, null=True) vul_update_time = models.DateTimeField(blank=True, null=True) - update_time = models.IntegerField(blank=True, null=True) - update_time_desc = models.IntegerField(blank=True, null=True) - create_time = models.IntegerField(blank=True, null=True) + update_time = models.IntegerField() + update_time_desc = models.IntegerField() + create_time = models.IntegerField() fix_plan = models.JSONField(blank=True, null=True, default=dict) poc = models.JSONField(blank=True, null=True, default=dict) descriptions = models.JSONField(blank=True, null=True, default=dict) @@ -54,17 +51,11 @@ class Meta: class IastAssetVulRelationMetaData(models.Model): - vul_asset_key = models.CharField(max_length=256, - blank=True, - primary_key=True) - vul_dependency_path = models.JSONField(blank=True, null=True, default=list) - effected_version_list = models.JSONField(blank=True, - null=True, - default=list) - fixed_version_list = models.JSONField(blank=True, null=True, default=list) - nearest_fixed_version = models.JSONField(blank=True, - null=True, - default=dict) + vul_asset_key = models.CharField(max_length=256, primary_key=True) + vul_dependency_path = models.JSONField(default=list) + effected_version_list = models.JSONField(default=list) + fixed_version_list = models.JSONField(default=list) + nearest_fixed_version = models.JSONField(default=dict) class Meta: managed = True @@ -84,8 +75,8 @@ class IastVulAssetRelation(models.Model): on_delete=models.DO_NOTHING, db_constraint=False, db_column='status_id') - is_del = models.SmallIntegerField(blank=True, null=False, default=0) - create_time = models.IntegerField(blank=True, null=True) + is_del = models.SmallIntegerField(default=0) + create_time = models.IntegerField() vul_asset_metadata = models.ForeignKey(IastAssetVulRelationMetaData, on_delete=models.DO_NOTHING, default="") @@ -97,8 +88,8 @@ class Meta: class IastAssetVulType(models.Model): - cwe_id = models.CharField(max_length=20, blank=True, null=True, default='') - name = models.CharField(max_length=100, blank=True, null=True, default='') + cwe_id = models.CharField(max_length=20, blank=True, default='') + name = models.CharField(max_length=100, blank=True, default='') class Meta: managed = get_managed() diff --git a/dongtai_common/models/asset_vul_v2.py b/dongtai_common/models/asset_vul_v2.py index bee19f0f5..8fb0e1583 100644 --- a/dongtai_common/models/asset_vul_v2.py +++ b/dongtai_common/models/asset_vul_v2.py @@ -1,38 +1,26 @@ -import uuid -from dongtai_common.models.agent import IastAgent -from django.core.cache import cache -from django_elasticsearch_dsl.search import Search -from dongtai_conf.settings import ASSET_VUL_INDEX -from django_elasticsearch_dsl import Document, fields -from django_elasticsearch_dsl.registries import registry from dongtai_common.models.assetv2 import AssetV2Global, AssetRiskLevel from django.db import models from dongtai_common.utils.settings import get_managed -from dongtai_common.models.vulnerablity import IastVulnerabilityStatus -from dongtai_common.models.vul_level import IastVulLevel class IastAssetVulV2(models.Model): - vul_name = models.CharField(max_length=255, blank=True, null=True) - vul_detail = models.TextField(blank=True, null=True) + vul_name = models.CharField(max_length=255) + vul_detail = models.TextField() # 漏洞类型等级 level = models.IntegerField(choices=AssetRiskLevel.choices, blank=True, db_column="level_id", default=AssetRiskLevel.LOW) - update_time = models.IntegerField(blank=True, null=True) - create_time = models.IntegerField(blank=True, null=True) - references = models.JSONField(blank=True, null=True, default=list) - change_time = models.IntegerField(blank=True, null=True) - published_time = models.IntegerField(blank=True, null=True) - vul_id = models.CharField(max_length=255, - blank=True, - null=True, - unique=True) - vul_type = models.JSONField(blank=True, null=True) - vul_codes = models.JSONField(blank=True, null=True) - affected_versions = models.JSONField(blank=True, null=True) - unaffected_versions = models.JSONField(blank=True, null=True) + update_time = models.IntegerField() + create_time = models.IntegerField() + references = models.JSONField(default=list) + change_time = models.IntegerField() + published_time = models.IntegerField() + vul_id = models.CharField(max_length=255, unique=True) + vul_type = models.JSONField() + vul_codes = models.JSONField() + affected_versions = models.JSONField() + unaffected_versions = models.JSONField() class Meta: managed = True diff --git a/dongtai_common/models/assetv2.py b/dongtai_common/models/assetv2.py index c7c4a3d70..c6227e493 100644 --- a/dongtai_common/models/assetv2.py +++ b/dongtai_common/models/assetv2.py @@ -5,27 +5,14 @@ # software: PyCharm # project: dongtai-models -import uuid import time -from django.core.cache import cache -from django_elasticsearch_dsl.search import Search -from dongtai_conf.settings import ASSET_INDEX -from django_elasticsearch_dsl import Document, fields -from django.db.models.fields.related import ForeignKey -from dongtai_web.utils import get_model_field -from django_elasticsearch_dsl.registries import registry -from django_elasticsearch_dsl import Document from django.db import models from django.utils.translation import gettext_lazy as _ -from dongtai_common.models import User -from dongtai_common.models.agent import IastAgent from dongtai_common.models.project import IastProject from dongtai_common.models.project_version import IastProjectVersion -from dongtai_common.models.vul_level import IastVulLevel from dongtai_common.utils.settings import get_managed from dongtai_common.models.department import Department -from dongtai_common.models.talent import Talent from django.db.models import IntegerChoices @@ -39,36 +26,26 @@ class AssetRiskLevel(IntegerChoices): class AssetV2(models.Model): id = models.BigAutoField(primary_key=True) - package_name = models.CharField(max_length=255, - blank=True, - null=True, - unique=True) - package_path = models.CharField(max_length=255, blank=True, null=True) - signature_algorithm = models.CharField(max_length=255, - blank=True, - null=True) - signature_value = models.CharField(max_length=255, blank=True, null=True) - dt = models.IntegerField(blank=True, - null=True, - default=lambda: int(time.time())) - version = models.CharField(max_length=255, blank=True, null=True) + package_name = models.CharField(max_length=255, unique=True) + package_path = models.CharField(max_length=255) + signature_algorithm = models.CharField(max_length=255) + signature_value = models.CharField(max_length=255) + dt = models.IntegerField(blank=True, default=lambda: int(time.time())) + version = models.CharField(max_length=255) project = models.ForeignKey(IastProject, on_delete=models.CASCADE, blank=True, - null=False, default=-1) project_version = models.ForeignKey(IastProjectVersion, on_delete=models.CASCADE, blank=True, - null=False, default=-1) # 部门id department = models.ForeignKey(Department, models.DO_NOTHING, blank=True, - null=True, default=-1) - language_id = models.IntegerField(default=1, blank=True, null=False) + language_id = models.IntegerField(default=1, blank=True) #is_reconized = models.IntegerField(blank=True, null=True) aql = models.ForeignKey('AssetV2Global', to_field='aql', @@ -83,7 +60,7 @@ class Meta: class AssetV2Global(models.Model): id = models.BigAutoField(primary_key=True) - package_name = models.CharField(max_length=255, blank=True, null=True) + package_name = models.CharField(max_length=255) package_fullname = models.ForeignKey( 'IastPackageGAInfo', on_delete=models.DO_NOTHING, @@ -91,26 +68,24 @@ class AssetV2Global(models.Model): db_column="package_fullname", to_field="package_fullname", ) - signature_algorithm = models.CharField(max_length=255, - blank=True, - null=True) - signature_value = models.CharField(max_length=255, blank=True, null=True) - version = models.CharField(max_length=255, blank=True, null=True) + signature_algorithm = models.CharField(max_length=255) + signature_value = models.CharField(max_length=255) + version = models.CharField(max_length=255) level = models.IntegerField( choices=AssetRiskLevel.choices, blank=True, default=AssetRiskLevel.NO_RISK, db_column="level_id", ) - vul_count = models.IntegerField(default=0, blank=True, null=True) - vul_critical_count = models.IntegerField(default=0, blank=True, null=False) - vul_high_count = models.IntegerField(default=0, blank=True, null=False) - vul_medium_count = models.IntegerField(default=0, blank=True, null=False) - vul_low_count = models.IntegerField(default=0, blank=True, null=False) - vul_info_count = models.IntegerField(default=0, blank=True, null=False) - license_list = models.JSONField(blank=True, null=True, default=list) - language_id = models.IntegerField(default=1, blank=True, null=False) - aql = models.CharField(max_length=255, blank=True, null=True, unique=True) + vul_count = models.IntegerField(default=0, blank=True) + vul_critical_count = models.IntegerField(default=0, blank=True) + vul_high_count = models.IntegerField(default=0, blank=True) + vul_medium_count = models.IntegerField(default=0, blank=True) + vul_low_count = models.IntegerField(default=0, blank=True) + vul_info_count = models.IntegerField(default=0, blank=True) + license_list = models.JSONField(blank=True, default=list) + language_id = models.IntegerField(default=1, blank=True) + aql = models.CharField(max_length=255, blank=True, unique=True) class Meta: managed = get_managed() @@ -141,7 +116,7 @@ class IastAssetLicense(models.Model): """ only for the filter """ - license_id = models.IntegerField(blank=True, null=True) + license_id = models.IntegerField() asset = models.ForeignKey(AssetV2Global, on_delete=models.DO_NOTHING, db_constraint=False, @@ -154,12 +129,9 @@ class Meta: class IastPackageGAInfo(models.Model): - package_fullname = models.CharField(max_length=255, - blank=True, - null=True, - unique=True) - affected_versions = models.JSONField(blank=True, null=True, default=list) - unaffected_versions = models.JSONField(blank=True, null=True, default=list) + package_fullname = models.CharField(max_length=255, unique=True) + affected_versions = models.JSONField(blank=True, default=list) + unaffected_versions = models.JSONField(blank=True, default=list) class Meta: managed = get_managed() diff --git a/dongtai_common/models/dast_integration.py b/dongtai_common/models/dast_integration.py index 283d673b0..c29c2796f 100644 --- a/dongtai_common/models/dast_integration.py +++ b/dongtai_common/models/dast_integration.py @@ -8,44 +8,26 @@ class IastDastIntegration(models.Model): - vul_name = models.CharField(max_length=255, blank=True, null=True) - detail = models.TextField( - blank=True, - null=True, - default='', - ) - vul_level = models.ForeignKey(IastVulLevel, - models.DO_NOTHING, - blank=True, - null=True) - payload = models.CharField( - max_length=255, - blank=True, - null=True, - default='', - ) - target = models.CharField(max_length=255, blank=True, null=True) - vul_type = models.CharField(max_length=255, blank=True, null=True) - dast_tag = models.CharField(max_length=255, blank=True, null=True) - request_messages = models.JSONField(null=False, default=list) - urls = models.JSONField(null=False, default=list) - create_time = models.IntegerField(default=lambda: int(time.time()), - blank=True, - null=True) - latest_time = models.IntegerField(default=lambda: int(time.time()), - blank=True, - null=True) + vul_name = models.CharField(max_length=255, blank=True) + detail = models.TextField(blank=True, default='') + vul_level = models.ForeignKey(IastVulLevel, models.DO_NOTHING, blank=True) + payload = models.CharField(max_length=255, blank=True, default='') + target = models.CharField(max_length=255, blank=True) + vul_type = models.CharField(max_length=255, blank=True) + dast_tag = models.CharField(max_length=255, blank=True) + request_messages = models.JSONField(default=list) + urls = models.JSONField(default=list) + create_time = models.IntegerField(default=lambda: int(time.time()), blank=True) + latest_time = models.IntegerField(default=lambda: int(time.time()), blank=True) project = models.ForeignKey(IastProject, on_delete=models.CASCADE, blank=True, - null=True, default=-1) project_version = models.ForeignKey(IastProjectVersion, on_delete=models.CASCADE, blank=True, - null=True, default=-1) - dongtai_vul_type = models.JSONField(null=False, default=list) + dongtai_vul_type = models.JSONField(default=list) class Meta: managed = get_managed() @@ -53,16 +35,14 @@ class Meta: class IastDastIntegrationRelation(models.Model): - dt_mark = models.CharField(max_length=255, blank=True, null=True) + dt_mark = models.CharField(max_length=255, blank=True) iastvul = models.ForeignKey(IastVulnerabilityModel, on_delete=models.CASCADE, blank=True, - null=True, default=-1) dastvul = models.ForeignKey(IastDastIntegration, on_delete=models.CASCADE, blank=True, - null=True, default=-1) class Meta: @@ -71,11 +51,10 @@ class Meta: class IastvulDtMarkRelation(models.Model): - dt_mark = models.CharField(max_length=255, blank=True, null=True) + dt_mark = models.CharField(max_length=255, blank=True) iastvul = models.ForeignKey(IastVulnerabilityModel, on_delete=models.CASCADE, blank=True, - null=True, default=-1) class Meta: @@ -84,11 +63,10 @@ class Meta: class DastvulDtMarkRelation(models.Model): - dt_mark = models.CharField(max_length=255, blank=True, null=True) + dt_mark = models.CharField(max_length=255, blank=True) dastvul = models.ForeignKey(IastDastIntegration, on_delete=models.CASCADE, blank=True, - null=True, default=-1) class Meta: diff --git a/dongtai_common/models/department.py b/dongtai_common/models/department.py index 11aa8270f..f9e0b9188 100644 --- a/dongtai_common/models/department.py +++ b/dongtai_common/models/department.py @@ -16,7 +16,7 @@ class IastDepartment(models.Model): - name = models.CharField(max_length=255, blank=True, null=True) + name = models.CharField(max_length=255, blank=True) class Meta: managed = get_managed() diff --git a/dongtai_common/models/deploy.py b/dongtai_common/models/deploy.py index 9dd7979ca..f306852a0 100644 --- a/dongtai_common/models/deploy.py +++ b/dongtai_common/models/deploy.py @@ -10,9 +10,9 @@ class IastDeployDesc(models.Model): - desc = models.TextField(blank=True, null=True) - middleware = models.CharField(max_length=255, blank=True, null=True) - language = models.CharField(max_length=255, blank=True, null=True) + desc = models.TextField() + middleware = models.CharField(max_length=255) + language = models.CharField(max_length=255) class Meta: managed = get_managed() diff --git a/dongtai_common/models/document.py b/dongtai_common/models/document.py index 07971705e..ca4193aec 100644 --- a/dongtai_common/models/document.py +++ b/dongtai_common/models/document.py @@ -3,9 +3,9 @@ class IastDocument(models.Model): - title = models.CharField(max_length=100, blank=True, null=True) - url = models.CharField(max_length=2000, blank=True, null=True) - language = models.CharField(max_length=100, blank=True, null=True) + title = models.CharField(max_length=100) + url = models.CharField(max_length=2000) + language = models.CharField(max_length=100) weight = models.IntegerField(default=0) class Meta: diff --git a/dongtai_common/models/engine_monitoring_indicators.py b/dongtai_common/models/engine_monitoring_indicators.py index c33b26174..5806354d5 100644 --- a/dongtai_common/models/engine_monitoring_indicators.py +++ b/dongtai_common/models/engine_monitoring_indicators.py @@ -11,12 +11,8 @@ class IastEnginMonitoringIndicators(models.Model): - key = models.CharField(max_length=100, - blank=True, - default='', - null=False, - unique=True) - name = models.CharField(max_length=100, blank=True, default='', null=False) + key = models.CharField(max_length=100, default='', unique=True) + name = models.CharField(max_length=100, default='') class Meta: managed = get_managed() diff --git a/dongtai_common/models/header_vulnerablity.py b/dongtai_common/models/header_vulnerablity.py index ab2a4f5a6..677169b66 100644 --- a/dongtai_common/models/header_vulnerablity.py +++ b/dongtai_common/models/header_vulnerablity.py @@ -1,10 +1,6 @@ from django.db import models from dongtai_common.models.agent import IastAgent -from dongtai_common.models.vul_level import IastVulLevel from dongtai_common.utils.settings import get_managed -from dongtai_common.models.strategy import IastStrategyModel -from dongtai_common.models.hook_type import HookType -from dongtai_common.models.server import IastServer from dongtai_common.models.project import IastProject from dongtai_common.models.project_version import IastProjectVersion from dongtai_common.models.agent_method_pool import MethodPool @@ -15,21 +11,15 @@ class IastHeaderVulnerability(models.Model): id = models.BigAutoField(primary_key=True) project = models.ForeignKey(IastProject, on_delete=models.DO_NOTHING, - blank=True, - null=True, default=-1, db_constraint=False) project_version = models.ForeignKey(IastProjectVersion, on_delete=models.DO_NOTHING, - blank=True, - null=True, default=-1, db_constraint=False) - url = models.CharField(max_length=255, default='', blank=True, null=True) + url = models.CharField(max_length=255, default='') vul = models.ForeignKey(IastVulnerabilityModel, on_delete=models.DO_NOTHING, - blank=True, - null=True, default=-1, db_constraint=False) @@ -39,25 +29,17 @@ class Meta: class IastHeaderVulnerabilityDetail(models.Model): - req_header = models.TextField( - blank=True, - null=True, - ) - res_header = models.TextField(blank=True, null=True) + req_header = models.TextField() + res_header = models.TextField(blank=True) agent = models.ForeignKey(IastAgent, models.DO_NOTHING, - blank=True, - null=True, db_constraint=False) method_pool = models.ForeignKey(MethodPool, models.DO_NOTHING, - blank=True, - null=True, db_constraint=False) header_vul = models.ForeignKey(IastHeaderVulnerability, models.DO_NOTHING, blank=True, - null=True, db_constraint=False) class Meta: diff --git a/dongtai_common/models/heartbeat.py b/dongtai_common/models/heartbeat.py index 6febe9a8f..75d57fcfe 100644 --- a/dongtai_common/models/heartbeat.py +++ b/dongtai_common/models/heartbeat.py @@ -17,25 +17,17 @@ class IastHeartbeat(models.Model): cpu = models.CharField(max_length=1000, blank=True, null=True) disk = models.CharField(max_length=1000, blank=True, null=True) req_count = models.IntegerField(default=0, blank=True, null=True) - dt = models.IntegerField(blank=True, null=True) - report_queue = models.PositiveIntegerField(default=0, - null=False, - blank=False) - method_queue = models.PositiveIntegerField(default=0, - null=False, - blank=False) - replay_queue = models.PositiveIntegerField(default=0, - null=False, - blank=False) + dt = models.IntegerField() + report_queue = models.PositiveIntegerField(default=0) + method_queue = models.PositiveIntegerField(default=0) + replay_queue = models.PositiveIntegerField(default=0) agent = models.ForeignKey( to=IastAgent, on_delete=models.DO_NOTHING, related_name='heartbeats', related_query_name='heartbeat', - verbose_name=_('agent'), - blank=True, - null=True + verbose_name=_('agent') ) class Meta: diff --git a/dongtai_common/models/hook_strategy.py b/dongtai_common/models/hook_strategy.py index ce71032d6..bf87f6803 100644 --- a/dongtai_common/models/hook_strategy.py +++ b/dongtai_common/models/hook_strategy.py @@ -31,28 +31,20 @@ class HookStrategy(models.Model): - value = models.CharField(max_length=255, blank=True, null=True) - source = models.CharField(max_length=255, blank=True, null=True) - target = models.CharField(max_length=255, blank=True, null=True) - inherit = models.CharField(max_length=255, blank=True, null=True) - track = models.CharField(max_length=5, blank=True, null=True) - create_time = models.IntegerField(blank=True, - null=True, - default=lambda: int(time())) - update_time = models.IntegerField(blank=True, - null=True, - default=lambda: int(time())) - created_by = models.IntegerField( - blank=True, - null=True, - ) - enable = models.IntegerField(blank=True, null=False, default=1) + value = models.CharField(max_length=255) + source = models.CharField(max_length=255, blank=True) + target = models.CharField(max_length=255, blank=True) + inherit = models.CharField(max_length=255) + track = models.CharField(max_length=5, blank=True) + create_time = models.IntegerField(default=lambda: int(time())) + update_time = models.IntegerField(default=lambda: int(time())) + created_by = models.IntegerField() + enable = models.IntegerField(default=1) language = models.ForeignKey(IastProgramLanguage, - blank=True, default='', on_delete=models.DO_NOTHING, db_constraint=False) - type = models.IntegerField(blank=True, null=True) + type = models.IntegerField() hooktype = models.ForeignKey( HookType, @@ -68,7 +60,6 @@ class HookStrategy(models.Model): ) strategy = models.ForeignKey( 'dongtai_common.IastStrategyModel', - blank=True, default=-1, on_delete=models.DO_NOTHING, db_column='strategy_id', @@ -76,17 +67,13 @@ class HookStrategy(models.Model): related_name="strategies", related_query_name="strategy", ) - system_type = models.IntegerField(blank=True, null=True, default=0) - ignore_blacklist = models.BooleanField(blank=True, - null=False, - default=False) - ignore_internal = models.BooleanField(blank=True, - null=False, - default=False) + system_type = models.IntegerField(blank=True, default=0) + ignore_blacklist = models.BooleanField(default=False) + ignore_internal = models.BooleanField(default=False) tags = models.JSONField(default=list) untags = models.JSONField(default=list) stack_blacklist = models.JSONField(default=list) - command = models.CharField(max_length=128, null=True, default="") + command = models.CharField(max_length=128, default="") class Meta: managed = get_managed() diff --git a/dongtai_common/models/hook_type.py b/dongtai_common/models/hook_type.py index 114e4d7b4..5e0c4ed16 100644 --- a/dongtai_common/models/hook_type.py +++ b/dongtai_common/models/hook_type.py @@ -7,18 +7,17 @@ from django.db import models from dongtai_common.utils.settings import get_managed from dongtai_common.models.program_language import IastProgramLanguage -from dongtai_common.models.user import User from time import time class HookType(models.Model): - type = models.IntegerField(blank=True, null=True) - name = models.CharField(max_length=255, blank=True, null=True) - value = models.CharField(max_length=255, blank=True, null=True) - enable = models.IntegerField(blank=True, null=True) - create_time = models.IntegerField(blank=True, null=True, default=time) - update_time = models.IntegerField(blank=True, null=True, default=time) - created_by = models.IntegerField(blank=True, null=True) + type = models.IntegerField() + name = models.CharField(max_length=255) + value = models.CharField(max_length=255) + enable = models.IntegerField(blank=True) + create_time = models.IntegerField(blank=True, null=True, default=lambda: int(time())) + update_time = models.IntegerField(blank=True, null=True, default=lambda: int(time())) + created_by = models.IntegerField(blank=True) language = models.ForeignKey(IastProgramLanguage, blank=True, default='', @@ -33,7 +32,7 @@ class HookType(models.Model): db_column='strategy_id', db_constraint=False, ) - system_type = models.IntegerField(blank=True, null=True, default=0) + system_type = models.IntegerField(blank=True, default=0) class Meta: managed = get_managed() diff --git a/dongtai_common/models/iast_vul_log.py b/dongtai_common/models/iast_vul_log.py index dc405cb77..947b9c00f 100644 --- a/dongtai_common/models/iast_vul_log.py +++ b/dongtai_common/models/iast_vul_log.py @@ -1,13 +1,11 @@ from django.db import models -from dongtai_common.models.agent import IastAgent from dongtai_common.utils.settings import get_managed from django.db.models import IntegerChoices from dongtai_common.models.user import User -from dongtai_common.models.asset import Asset from dongtai_common.models.vulnerablity import IastVulnerabilityModel from time import time -from dongtai_common.models.asset_vul import IastAssetVul, IastVulAssetRelation +from dongtai_common.models.asset_vul import IastAssetVul class MessageTypeChoices(IntegerChoices): @@ -18,10 +16,10 @@ class MessageTypeChoices(IntegerChoices): class IastVulLog(models.Model): - msg_type = models.IntegerField(blank=True, null=True) - msg = models.TextField(blank=True, null=True) - meta_data = models.JSONField(blank=True, null=True) - datetime = models.IntegerField(blank=True, null=True, default=time) + msg_type = models.IntegerField() + msg = models.TextField() + meta_data = models.JSONField() + datetime = models.IntegerField(default=lambda: int(time())) vul = models.ForeignKey(IastVulnerabilityModel, models.DO_NOTHING, default=-1, diff --git a/dongtai_common/models/message.py b/dongtai_common/models/message.py index 37295c84b..c58755611 100644 --- a/dongtai_common/models/message.py +++ b/dongtai_common/models/message.py @@ -13,7 +13,7 @@ class IastMessageType(models.Model): - name = models.CharField(max_length=100, blank=True, null=False, default='') + name = models.CharField(max_length=100, default='') class Meta: managed = get_managed() @@ -21,17 +21,11 @@ class Meta: class IastMessage(models.Model): - message = models.CharField(max_length=512, - blank=True, - null=False, - default='') - relative_url = models.CharField(max_length=512, - blank=True, - null=False, - default='') - create_time = models.IntegerField(blank=True, default=lambda: int(time())) - read_time = models.IntegerField(blank=True, default=0) - is_read = models.IntegerField(blank=True, null=True, default=0) + message = models.CharField(max_length=512, default='') + relative_url = models.CharField(max_length=512, default='') + create_time = models.IntegerField(default=lambda: int(time())) + read_time = models.IntegerField(default=0) + is_read = models.IntegerField(default=0) message_type = models.ForeignKey(IastMessageType, on_delete=models.DO_NOTHING, db_constraint=False, diff --git a/dongtai_common/models/notify_config.py b/dongtai_common/models/notify_config.py index 344d77d1c..04a54690a 100644 --- a/dongtai_common/models/notify_config.py +++ b/dongtai_common/models/notify_config.py @@ -35,11 +35,11 @@ class IastNotifyConfig(models.Model): WEIXIN = WEIXIN NOTIFY_TYPE_CHOICES = NOTIFY_TYPE_CHOICES - notify_type = models.SmallIntegerField(blank=True, null=True, choices=NOTIFY_TYPE_CHOICES) - notify_meta_data = models.TextField(blank=True, null=True) # This field type is a guess. - user = models.ForeignKey(to=User, on_delete=models.DO_NOTHING, blank=True, null=True) - test_result = models.SmallIntegerField(blank=True, null=True, default=0) - create_time = models.IntegerField(blank=True, null=True) + notify_type = models.SmallIntegerField(choices=NOTIFY_TYPE_CHOICES) + notify_meta_data = models.TextField() # This field type is a guess. + user = models.ForeignKey(to=User, on_delete=models.DO_NOTHING, ) + test_result = models.SmallIntegerField(default=0) + create_time = models.IntegerField() class Meta: managed = get_managed() diff --git a/dongtai_common/models/profile.py b/dongtai_common/models/profile.py index 7610c3944..32147da4b 100644 --- a/dongtai_common/models/profile.py +++ b/dongtai_common/models/profile.py @@ -4,7 +4,7 @@ class IastProfile(models.Model): key = models.CharField(max_length=100) - value = models.CharField(max_length=100, blank=True, null=True) + value = models.CharField(max_length=100) class Meta: managed = get_managed() diff --git a/dongtai_common/models/program_language.py b/dongtai_common/models/program_language.py index 0e55f2beb..86b5546e3 100644 --- a/dongtai_common/models/program_language.py +++ b/dongtai_common/models/program_language.py @@ -12,7 +12,7 @@ class IastProgramLanguage(models.Model): - name = models.CharField(max_length=255, blank=True) + name = models.CharField(max_length=255) class Meta: managed = get_managed() diff --git a/dongtai_common/models/project.py b/dongtai_common/models/project.py index 615afc59b..0a739249f 100644 --- a/dongtai_common/models/project.py +++ b/dongtai_common/models/project.py @@ -21,19 +21,11 @@ class VulValidation(models.IntegerChoices): class IastProjectTemplate(models.Model): - template_name = models.CharField(max_length=255, blank=True, null=True) - latest_time = models.IntegerField(default=lambda: int(time.time()), - blank=True, - null=True) - user = models.ForeignKey(User, models.DO_NOTHING, blank=True, null=True) - scan = models.ForeignKey(IastStrategyUser, - models.DO_NOTHING, - blank=True, - null=True) - vul_validation = models.IntegerField(default=0, - blank=True, - null=False, - choices=VulValidation.choices) + template_name = models.CharField(max_length=255) + latest_time = models.IntegerField(default=lambda: int(time.time())) + user = models.ForeignKey(User, models.DO_NOTHING) + scan = models.ForeignKey(IastStrategyUser, models.DO_NOTHING) + vul_validation = models.IntegerField(default=0, choices=VulValidation.choices) is_system = models.IntegerField(default=0) data_gather = models.JSONField(default=dict) data_gather_is_followglobal = models.IntegerField(default=1) @@ -56,27 +48,19 @@ def to_full_project_args(self): } class IastProject(models.Model): - name = models.CharField(max_length=255, blank=True, null=True) - mode = models.CharField(default="插桩模式", - max_length=255, - blank=True, - null=True) + name = models.CharField(max_length=255) + mode = models.CharField(default="插桩模式", max_length=255) vul_count = models.PositiveIntegerField(blank=True, null=True) agent_count = models.IntegerField(blank=True, null=True) - latest_time = models.IntegerField(default=lambda: int(time.time()), - blank=True, - null=True) - user = models.ForeignKey(User, models.DO_NOTHING, blank=True, null=True) + latest_time = models.IntegerField(default=lambda: int(time.time())) + user = models.ForeignKey(User, models.DO_NOTHING) # openapi服务不必使用该字段 scan = models.ForeignKey(IastStrategyUser, models.DO_NOTHING, blank=True, null=True) - vul_validation = models.IntegerField(default=0, - blank=True, - null=False, - choices=VulValidation.choices) + vul_validation = models.IntegerField(default=0, choices=VulValidation.choices) base_url = models.CharField(max_length=255, blank=True, default='') test_req_header_key = models.CharField(max_length=511, blank=True, @@ -84,17 +68,11 @@ class IastProject(models.Model): test_req_header_value = models.CharField(max_length=511, blank=True, default='') - data_gather = models.JSONField() + data_gather = models.JSONField(null=True) data_gather_is_followglobal = models.IntegerField(default=1) blacklist_is_followglobal = models.IntegerField(default=1) - department = models.ForeignKey(Department, - models.DO_NOTHING, - blank=True, - null=True) - template = models.ForeignKey(IastProjectTemplate, - models.DO_NOTHING, - blank=True, - null=True) + department = models.ForeignKey(Department, models.DO_NOTHING) + template = models.ForeignKey(IastProjectTemplate, models.DO_NOTHING) enable_log = models.BooleanField(null=True) log_level = models.CharField(max_length=511, null=True) diff --git a/dongtai_common/models/project_report.py b/dongtai_common/models/project_report.py index fdb41cdd5..3a3e8a27e 100644 --- a/dongtai_common/models/project_report.py +++ b/dongtai_common/models/project_report.py @@ -5,11 +5,9 @@ # software: PyCharm # project: dongtai-models from django.db import models -from django.utils.translation import gettext_lazy as _ from dongtai_common.models import User from dongtai_common.models.project import IastProject -from dongtai_common.models.server import IastServer from dongtai_common.utils.settings import get_managed ORDER_TYPE_REPORT = { @@ -20,20 +18,20 @@ class ProjectReport(models.Model): user = models.ForeignKey(User, models.DO_NOTHING) - project = models.ForeignKey(IastProject, models.DO_NOTHING, blank=True, null=True) - type = models.CharField(max_length=10, blank=True, null=True) - language = models.CharField(max_length=10, blank=True, null=True) - status = models.IntegerField(default=0, null=False) - path = models.CharField(default='', max_length=255, blank=True, null=True) + project = models.ForeignKey(IastProject, models.DO_NOTHING) + type = models.CharField(max_length=10) + language = models.CharField(max_length=10) + status = models.IntegerField(default=0) + path = models.CharField(default='', max_length=255, blank=True) file = models.BinaryField(blank=True, null=True) - create_time = models.IntegerField(default=0, null=False) - is_del = models.SmallIntegerField(default=0, null=False) + create_time = models.IntegerField(default=0) + is_del = models.SmallIntegerField(default=0) level_png = models.CharField(default='', max_length=255, blank=True, null=True) trend_png = models.CharField(default='', max_length=255, blank=True, null=True) version_str = models.CharField(default='', max_length=255, blank=True, null=True) vul_type_str = models.CharField(default='', max_length=255, blank=True, null=True) sca_type_str = models.TextField(default='', blank=True, null=True) - vul_id = models.IntegerField(blank=True, null=True, default=0) + vul_id = models.IntegerField(default=0) report_name = models.CharField(default='', max_length=255, blank=True, null=True) class Meta: diff --git a/dongtai_common/models/project_version.py b/dongtai_common/models/project_version.py index c49fe47a7..00ea1e0fa 100644 --- a/dongtai_common/models/project_version.py +++ b/dongtai_common/models/project_version.py @@ -13,18 +13,15 @@ class IastProjectVersion(models.Model): - version_name = models.CharField(max_length=255, blank=True, null=True) - description = models.TextField(blank=True, null=True) - current_version = models.PositiveSmallIntegerField(blank=True, default=0) - status = models.PositiveSmallIntegerField(blank=True, null=True) + version_name = models.CharField(max_length=255) + description = models.TextField(blank=True) + current_version = models.PositiveSmallIntegerField(default=0) + status = models.PositiveSmallIntegerField() create_time = models.IntegerField(_('create time'), - default=lambda: int(time.time()), - blank=True) - update_time = models.IntegerField(_('update time'), - default=lambda: int(time.time()), - blank=True) - user = models.ForeignKey(User, models.DO_NOTHING, blank=True, null=True) - project = models.ForeignKey(IastProject, models.DO_NOTHING, blank=True, null=True) + default=lambda: int(time.time())) + update_time = models.IntegerField(_('update time'), default=lambda: int(time.time())) + user = models.ForeignKey(User, models.DO_NOTHING) + project = models.ForeignKey(IastProject, models.DO_NOTHING) class Meta: managed = get_managed() diff --git a/dongtai_common/models/recognize_rule.py b/dongtai_common/models/recognize_rule.py index 2d3f36cc6..183dba58d 100644 --- a/dongtai_common/models/recognize_rule.py +++ b/dongtai_common/models/recognize_rule.py @@ -1,6 +1,5 @@ from django.db import models from dongtai_common.models.project import IastProject -from django.db.models import IntegerChoices from dongtai_common.utils.settings import get_managed diff --git a/dongtai_common/models/replay_method_pool.py b/dongtai_common/models/replay_method_pool.py index 256ab5266..bcd4b7b63 100644 --- a/dongtai_common/models/replay_method_pool.py +++ b/dongtai_common/models/replay_method_pool.py @@ -12,25 +12,25 @@ class IastAgentMethodPoolReplay(models.Model): id = models.BigAutoField(primary_key=True) - agent = models.ForeignKey(IastAgent, models.DO_NOTHING, blank=True, null=True) - url = models.CharField(max_length=2000, blank=True, null=True) - uri = models.CharField(max_length=2000, blank=True, null=True) - http_method = models.CharField(max_length=10, blank=True, null=True) - http_scheme = models.CharField(max_length=20, blank=True, null=True) - http_protocol = models.CharField(max_length=255, blank=True, null=True) - req_header = models.CharField(max_length=2000, blank=True, null=True) + agent = models.ForeignKey(IastAgent, models.DO_NOTHING) + url = models.CharField(max_length=2000) + uri = models.CharField(max_length=2000) + http_method = models.CharField(max_length=10) + http_scheme = models.CharField(max_length=20) + http_protocol = models.CharField(max_length=255) + req_header = models.CharField(max_length=2000) req_params = models.CharField(max_length=2000, blank=True, null=True) req_data = models.CharField(max_length=4000, blank=True, null=True) res_header = models.CharField(max_length=1000, blank=True, null=True) res_body = models.CharField(max_length=1000, blank=True, null=True) context_path = models.CharField(max_length=255, blank=True, null=True) - method_pool = models.TextField(blank=True, null=True) # This field type is a guess. - clent_ip = models.CharField(max_length=255, blank=True, null=True) - create_time = models.IntegerField(blank=True, null=True) - update_time = models.IntegerField(blank=True, null=True) - replay_id = models.IntegerField(blank=True, null=True) - replay_type = models.IntegerField(blank=True, null=True) - relation_id = models.IntegerField(blank=True, null=True) + method_pool = models.TextField() # This field type is a guess. + clent_ip = models.CharField(max_length=255) + create_time = models.IntegerField() + update_time = models.IntegerField() + replay_id = models.IntegerField() + replay_type = models.IntegerField() + relation_id = models.IntegerField() class Meta: managed = get_managed() diff --git a/dongtai_common/models/replay_queue.py b/dongtai_common/models/replay_queue.py index d11e682cd..ca1787353 100644 --- a/dongtai_common/models/replay_queue.py +++ b/dongtai_common/models/replay_queue.py @@ -6,7 +6,6 @@ from django.db import models from dongtai_common.models.agent import IastAgent -from dongtai_common.models.vulnerablity import IastVulnerabilityModel from dongtai_common.utils.settings import get_managed from dongtai_common.models.vul_recheck_payload import IastVulRecheckPayload diff --git a/dongtai_common/models/res_header.py b/dongtai_common/models/res_header.py index 73e59d276..1e43fc80c 100644 --- a/dongtai_common/models/res_header.py +++ b/dongtai_common/models/res_header.py @@ -18,12 +18,8 @@ class HeaderType(models.IntegerChoices): class ProjectSaasMethodPoolHeader(models.Model): - key = models.CharField(max_length=255, blank=True, null=False) - agent = models.ForeignKey(IastAgent, - models.DO_NOTHING, - blank=True, - null=True, - db_constraint=False) + key = models.CharField(max_length=255, blank=True) + agent = models.ForeignKey(IastAgent, models.DO_NOTHING, db_constraint=False) header_type = models.IntegerField(choices=HeaderType.choices, default=0) class Meta: diff --git a/dongtai_common/models/sensitive_info.py b/dongtai_common/models/sensitive_info.py index 94f4b97f2..68fbf8c4d 100644 --- a/dongtai_common/models/sensitive_info.py +++ b/dongtai_common/models/sensitive_info.py @@ -13,7 +13,7 @@ class IastPatternType(models.Model): - name = models.CharField(blank=True, default=None, max_length=255) + name = models.CharField(max_length=255) id = models.IntegerField(default=0, db_column='value') logi_id = models.BigAutoField(primary_key=True, db_column='id') @@ -22,20 +22,12 @@ class Meta: class IastSensitiveInfoRule(models.Model): - user = models.ForeignKey(User, models.DO_NOTHING, blank=True, null=True) - strategy = models.ForeignKey(IastStrategyModel, - models.DO_NOTHING, - blank=True, - null=True) - pattern_type = models.ForeignKey(IastPatternType, - models.DO_NOTHING, - blank=True, - default=None) - pattern = models.CharField(blank=True, default=None, max_length=255) - status = models.IntegerField(blank=True, default=None) - latest_time = models.IntegerField(default=lambda: int(time.time()), - blank=True, - null=True) + user = models.ForeignKey(User, models.DO_NOTHING) + strategy = models.ForeignKey(IastStrategyModel, models.DO_NOTHING) + pattern_type = models.ForeignKey(IastPatternType, models.DO_NOTHING) + pattern = models.CharField(default=None, max_length=255) + status = models.IntegerField(default=None) + latest_time = models.IntegerField(default=lambda: int(time.time())) class Meta: db_table = 'iast_sensitive_info_rule' diff --git a/dongtai_common/models/server.py b/dongtai_common/models/server.py index c619fea9f..2d0468f41 100644 --- a/dongtai_common/models/server.py +++ b/dongtai_common/models/server.py @@ -9,27 +9,24 @@ class IastServer(models.Model): - hostname = models.CharField(max_length=255, blank=True, null=True) - ip = models.CharField(max_length=255, blank=True, null=True) - port = models.IntegerField(blank=True, null=True) + hostname = models.CharField(max_length=255) + ip = models.CharField(max_length=255, blank=True) + port = models.IntegerField() environment = models.TextField(blank=True, null=True) - path = models.CharField(max_length=255, blank=True, null=True) - status = models.CharField(max_length=255, blank=True, null=True) - container = models.CharField(max_length=255, blank=True, null=True) - container_path = models.CharField(max_length=255, blank=True, null=True) - cluster_name = models.CharField(max_length=255, blank=True, null=True) - cluster_version = models.CharField(max_length=100, blank=True, null=True) - command = models.TextField(blank=True, null=True) - env = models.CharField(max_length=255, blank=True, null=True) - runtime = models.CharField(max_length=255, blank=True, null=True) - create_time = models.IntegerField(blank=True, null=True) - update_time = models.IntegerField(blank=True, null=True) - network = models.CharField(max_length=255, blank=True, null=True) - protocol = models.CharField(max_length=255, - blank=True, - null=True, - default='') - pid = models.IntegerField(blank=True, null=True) + path = models.CharField(max_length=255, blank=True) + status = models.CharField(max_length=255) + container = models.CharField(max_length=255, blank=True) + container_path = models.CharField(max_length=255, blank=True) + cluster_name = models.CharField(max_length=255, blank=True) + cluster_version = models.CharField(max_length=100, blank=True) + command = models.TextField(blank=True) + env = models.CharField(max_length=255) + runtime = models.CharField(max_length=255, blank=True) + create_time = models.IntegerField() + update_time = models.IntegerField() + network = models.CharField(max_length=255, blank=True) + protocol = models.CharField(max_length=255, blank=True, default='') + pid = models.IntegerField(blank=True) ipaddresslist = models.JSONField(null=False, default=list) class Meta: diff --git a/dongtai_common/models/strategy.py b/dongtai_common/models/strategy.py index 644be81db..a16203c63 100644 --- a/dongtai_common/models/strategy.py +++ b/dongtai_common/models/strategy.py @@ -8,19 +8,19 @@ class IastStrategyModel(models.Model): - user = models.ForeignKey(User, models.DO_NOTHING, blank=True, null=True) - vul_type = models.CharField(max_length=255, blank=True, null=True) - level = models.ForeignKey(IastVulLevel, models.DO_NOTHING, blank=True, null=True) - state = models.CharField(max_length=255, blank=True, null=True) - dt = models.IntegerField(blank=True, null=True, default=time) - vul_name = models.CharField(max_length=255, blank=True, null=True) - vul_desc = models.TextField(blank=True, null=True) - vul_fix = models.TextField(blank=True, null=True) + user = models.ForeignKey(User, models.DO_NOTHING) + vul_type = models.CharField(max_length=255) + level = models.ForeignKey(IastVulLevel, models.DO_NOTHING) + state = models.CharField(max_length=255) + dt = models.IntegerField(blank=True, default=lambda: int(time())) + vul_name = models.CharField(max_length=255) + vul_desc = models.TextField() + vul_fix = models.TextField(blank=True) hook_type = models.ForeignKey(HookType, models.DO_NOTHING, blank=True, null=True) - system_type = models.IntegerField(blank=True, null=True, default=0) + system_type = models.IntegerField(blank=True, default=0) class Meta: managed = get_managed() diff --git a/dongtai_common/models/strategy_user.py b/dongtai_common/models/strategy_user.py index 3b74d231c..98781007b 100644 --- a/dongtai_common/models/strategy_user.py +++ b/dongtai_common/models/strategy_user.py @@ -7,10 +7,10 @@ class IastStrategyUser(models.Model): id = models.BigAutoField(primary_key=True) - name = models.CharField(max_length=200, blank=True, null=True) - content = models.TextField(blank=True, null=True) - user = models.ForeignKey(User, models.DO_NOTHING, blank=True, null=True) - status = models.IntegerField(blank=True, null=True) + name = models.CharField(max_length=200) + content = models.TextField(blank=True) + user = models.ForeignKey(User, models.DO_NOTHING) + status = models.IntegerField(blank=True) created_at = models.DateTimeField(verbose_name="创建时间", auto_now_add=True) department = models.ForeignKey(Department, models.DO_NOTHING, diff --git a/dongtai_common/models/talent.py b/dongtai_common/models/talent.py index b80ed55a0..586ee7019 100644 --- a/dongtai_common/models/talent.py +++ b/dongtai_common/models/talent.py @@ -21,9 +21,9 @@ class Talent(models.Model): 'unique': _("A talent with that talent name already exists."), }, ) - create_time = models.IntegerField(blank=True, null=True) - update_time = models.IntegerField(blank=True, null=True) - created_by = models.IntegerField(blank=True, null=True) + create_time = models.IntegerField() + update_time = models.IntegerField() + created_by = models.IntegerField() is_active = models.BooleanField( _('active'), default=True, diff --git a/dongtai_common/models/tests.py b/dongtai_common/models/tests.py index 7077f78a0..67e620ea8 100644 --- a/dongtai_common/models/tests.py +++ b/dongtai_common/models/tests.py @@ -1,6 +1,4 @@ from django.test import TestCase -import os -import json from dongtai_common.models.url_blacklist import ( IastAgentBlackRule, create_blacklist_rule, diff --git a/dongtai_common/models/url_blacklist.py b/dongtai_common/models/url_blacklist.py index 08d0b2953..2aaad3030 100644 --- a/dongtai_common/models/url_blacklist.py +++ b/dongtai_common/models/url_blacklist.py @@ -3,7 +3,6 @@ from dongtai_common.utils.settings import get_managed from django.db.models import IntegerChoices from django.utils.translation import gettext_lazy as _ -from time import time from typing import List, Dict @@ -57,26 +56,12 @@ def to_full_rule(self) -> List[Dict]: class IastAgentBlackRuleDetail(models.Model): - target_type = models.IntegerField( - choices=TargetType.choices, - blank=True, - null=True, - ) + target_type = models.IntegerField(choices=TargetType.choices) rule = models.ForeignKey(IastAgentBlackRule, models.DO_NOTHING) - operator = models.IntegerField(choices=TargetOperator.choices, - blank=True, - null=True) + operator = models.IntegerField(choices=TargetOperator.choices) value = models.CharField(max_length=512, default="", null=False) - create_time = models.DateTimeField( - blank=True, - null=True, - auto_now_add=True, - ) - update_time = models.DateTimeField( - blank=True, - null=True, - auto_now=True, - ) + create_time = models.DateTimeField(auto_now_add=True) + update_time = models.DateTimeField(auto_now=True) class Meta: managed = get_managed() diff --git a/dongtai_common/models/user.py b/dongtai_common/models/user.py index 1a060255f..8499d9efa 100644 --- a/dongtai_common/models/user.py +++ b/dongtai_common/models/user.py @@ -11,7 +11,6 @@ from django.utils.translation import gettext_lazy as _ from dongtai_common.models.department import Department -from dongtai_common.utils.settings import get_managed class PermissionsMixin(models.Model): diff --git a/dongtai_common/models/vul_level.py b/dongtai_common/models/vul_level.py index 1558034ca..91881acae 100644 --- a/dongtai_common/models/vul_level.py +++ b/dongtai_common/models/vul_level.py @@ -9,9 +9,9 @@ class IastVulLevel(models.Model): - name = models.CharField(max_length=255, blank=True, null=True) - name_value = models.CharField(max_length=255, blank=True, null=True) - name_type = models.CharField(max_length=255, blank=True, null=True) + name = models.CharField(max_length=255) + name_value = models.CharField(max_length=255) + name_type = models.CharField(max_length=255) class Meta: managed = get_managed() diff --git a/dongtai_common/models/vul_recheck_payload.py b/dongtai_common/models/vul_recheck_payload.py index feb59f7e6..1ad4e81e7 100644 --- a/dongtai_common/models/vul_recheck_payload.py +++ b/dongtai_common/models/vul_recheck_payload.py @@ -1,9 +1,6 @@ from django.db import models from dongtai_common.models import User -from dongtai_common.utils.settings import get_managed from time import time -from django.db.models import IntegerChoices -from django.utils.translation import gettext_lazy as _ from dongtai_common.models.strategy import IastStrategyModel diff --git a/dongtai_common/models/vulnerablity.py b/dongtai_common/models/vulnerablity.py index a4c972b20..3af30ab1c 100644 --- a/dongtai_common/models/vulnerablity.py +++ b/dongtai_common/models/vulnerablity.py @@ -25,30 +25,27 @@ class Meta: class IastVulnerabilityModel(models.Model): id = models.BigAutoField(primary_key=True) - search_keywords = models.CharField(max_length=1000, blank=True, null=True) + search_keywords = models.CharField(max_length=1000, blank=True) level = models.ForeignKey(IastVulLevel, models.DO_NOTHING, - blank=True, - null=True) - url = models.CharField(max_length=2000, blank=True, null=True) - uri = models.CharField(max_length=255, blank=True, null=True) + blank=True) + url = models.CharField(max_length=2000, blank=True) + uri = models.CharField(max_length=255, blank=True) pattern_uri = models.CharField(max_length=255, blank=True, null=True) # 模糊搜索 全文索引 查询 vul_title = models.CharField(max_length=255, blank=True, - null=True, default="") - http_method = models.CharField(max_length=10, blank=True, null=True) - http_scheme = models.CharField(max_length=255, blank=True, null=True) - http_protocol = models.CharField(max_length=255, blank=True, null=True) - req_header = models.TextField(blank=True, null=True) + http_method = models.CharField(max_length=10, blank=True) + http_scheme = models.CharField(max_length=255, blank=True) + http_protocol = models.CharField(max_length=255, blank=True) + req_header = models.TextField(blank=True) req_params = models.CharField(max_length=2000, blank=True, - null=True, default="") - req_data = models.TextField(blank=True, null=True) - res_header = models.TextField(blank=True, null=True) - res_body = models.TextField(blank=True, null=True) + req_data = models.TextField(blank=True, ) + res_header = models.TextField(blank=True) + res_body = models.TextField(blank=True) full_stack = models.TextField(blank=True, null=True) top_stack = models.CharField(max_length=255, blank=True, null=True) bottom_stack = models.CharField(max_length=255, blank=True, null=True) @@ -56,22 +53,21 @@ class IastVulnerabilityModel(models.Model): taint_position = models.CharField(max_length=255, blank=True, null=True) agent = models.ForeignKey(IastAgent, models.DO_NOTHING, - blank=True, - null=True) + blank=True) language = models.CharField(max_length=10, blank=True, null=True) - context_path = models.CharField(max_length=255, blank=True, null=True) - counts = models.IntegerField(blank=True, null=True) - first_time = models.IntegerField(blank=True, null=True) - latest_time = models.IntegerField(blank=True, null=True) - latest_time_desc = models.IntegerField(blank=True, null=True, default=0) - level_id_desc = models.SmallIntegerField(blank=True, null=True, default=0) - client_ip = models.CharField(max_length=255, blank=True, null=True) + context_path = models.CharField(max_length=255, blank=True) + counts = models.IntegerField(blank=True) + first_time = models.IntegerField(blank=True) + latest_time = models.IntegerField(blank=True) + latest_time_desc = models.IntegerField(blank=True, default=0) + level_id_desc = models.SmallIntegerField(blank=True, default=0) + client_ip = models.CharField(max_length=255, blank=True) param_name = models.CharField(max_length=255, blank=True, null=True, default='') - is_del = models.SmallIntegerField(blank=True, null=True, default=0) - method_pool_id = models.IntegerField(default=-1, blank=True, null=True) + is_del = models.SmallIntegerField(blank=True, default=0) + method_pool_id = models.IntegerField(default=-1, blank=True) strategy = models.ForeignKey(IastStrategyModel, on_delete=models.DO_NOTHING, db_constraint=False, @@ -83,21 +79,19 @@ class IastVulnerabilityModel(models.Model): status = models.ForeignKey(IastVulnerabilityStatus, on_delete=models.DO_NOTHING, db_constraint=False, - db_column='status_id') + db_column='status_id', + null=True) project = models.ForeignKey(IastProject, on_delete=models.CASCADE, blank=True, - null=True, default=-1) project_version = models.ForeignKey(IastProjectVersion, on_delete=models.CASCADE, blank=True, - null=True, default=-1) server = models.ForeignKey(IastServer, on_delete=models.CASCADE, blank=True, - null=True, default=-1) class Meta: diff --git a/dongtai_engine/plugins/data_clean.py b/dongtai_engine/plugins/data_clean.py index 3df2d97ca..d370bda6d 100644 --- a/dongtai_engine/plugins/data_clean.py +++ b/dongtai_engine/plugins/data_clean.py @@ -1,10 +1,5 @@ from dongtai_common.models.agent import IastAgent -from dongtai_common.models.vulnerablity import IastVulnerabilityModel, IastVulnerabilityDocument from celery import shared_task -from django.apps import apps -from django.db import transaction -from dongtai_common.models.asset import Asset, IastAssetDocument -from dongtai_common.models.asset_vul import IastVulAssetRelation, IastAssetVulnerabilityDocument from dongtai_common.models.agent_method_pool import MethodPool from time import time from celery.apps.worker import logger @@ -61,11 +56,10 @@ def data_cleanup(days: int): first_id = MethodPool.objects.filter( update_time__lte=delete_time_stamp).order_by('id').values_list( 'id', flat=True).first() - if not any([latest_id, first_id]): + if not any((latest_id, first_id)) or not isinstance(latest_id, int) or not isinstance(first_id, int): logger.info("no data for clean up") - if all([latest_id, first_id]): - assert isinstance(latest_id, int) - assert isinstance(first_id, int) + return + if all((latest_id, first_id)): batch_clean(latest_id, first_id, 10000) # qs = MethodPool.objects.filter(pk__lte=latest_id) # qs._raw_delete(qs.db) diff --git a/dongtai_engine/plugins/strategy_headers.py b/dongtai_engine/plugins/strategy_headers.py index e2ef0da95..e73b239b8 100644 --- a/dongtai_engine/plugins/strategy_headers.py +++ b/dongtai_engine/plugins/strategy_headers.py @@ -5,21 +5,18 @@ # project: DongTai-engine import uuid from django.core.cache import cache -import random import time from http.client import HTTPResponse from http.client import BadStatusLine from io import BytesIO from celery.apps.worker import logger -from django.db.models import Q -from dongtai_common.models.project import IastProject from dongtai_common.models.strategy import IastStrategyModel from dongtai_common.models.vulnerablity import IastVulnerabilityModel from dongtai_common.utils import const from dongtai_engine.plugins import is_strategy_enable -from dongtai_web.vul_log.vul_log import log_vul_found, log_recheck_vul +from dongtai_web.vul_log.vul_log import log_vul_found from dongtai_common.models.header_vulnerablity import IastHeaderVulnerability, IastHeaderVulnerabilityDetail from django.db import IntegrityError from dongtai_engine.plugins.project_time_update import project_time_stamp_update @@ -200,7 +197,7 @@ def save_vul(vul_type, method_pool, position=None, data=None): server_id=method_pool.agent.server_id, ) log_vul_found(vul.agent.user_id, vul.agent.bind_project.name, - vul.agent.bind_project_id, vul.id, vul.strategy.vul_name) + vul.agent.bind_project_id, vul.id, vul.strategy.vul_name) # type: ignore cache.delete(cache_key) header_vul = None if not IastHeaderVulnerability.objects.filter( diff --git a/dongtai_engine/signals/handlers/vul_handler.py b/dongtai_engine/signals/handlers/vul_handler.py index c11b9ca7d..744261eba 100644 --- a/dongtai_engine/signals/handlers/vul_handler.py +++ b/dongtai_engine/signals/handlers/vul_handler.py @@ -4,13 +4,10 @@ # datetime: 2021/4/30 下午3:00 # project: dongtai-engine import json -import random import time -import requests from celery.apps.worker import logger from django.dispatch import receiver -from dongtai_common.models.agent import IastAgent from dongtai_common.models.project import IastProject, VulValidation from dongtai_common.models.replay_queue import IastReplayQueue from dongtai_common.models.vulnerablity import IastVulnerabilityModel @@ -22,7 +19,6 @@ from django.db.models import Q from dongtai_engine.signals.handlers.parse_param_name import parse_target_values_from_vul_stack from typing import List, Optional, Callable -import json from collections import defaultdict from dongtai_common.models.profile import IastProfile from dongtai_engine.plugins.project_time_update import project_time_stamp_update @@ -358,8 +354,8 @@ def save_vul(vul_meta, vul_level, strategy_id, vul_stack, top_stack, language=vul_meta.agent.language, server_id=vul_meta.agent.server_id, ) - log_vul_found(vul.agent.user_id, vul.agent.bind_project.name, - vul.agent.bind_project_id, vul.id, vul.strategy.vul_name) + log_vul_found(vul.agent.user_id, vul.agent.bind_project.name, # type: ignore + vul.agent.bind_project_id, vul.id, vul.strategy.vul_name) # type: ignore cache.delete(cache_key) #delete if exists more than one departured use redis lock #IastVulnerabilityModel.objects.filter( diff --git a/dongtai_protocol/report/handler/hardencode_vul_handler.py b/dongtai_protocol/report/handler/hardencode_vul_handler.py index 7ba1355be..78b036dde 100644 --- a/dongtai_protocol/report/handler/hardencode_vul_handler.py +++ b/dongtai_protocol/report/handler/hardencode_vul_handler.py @@ -8,13 +8,10 @@ import json import logging -import random import time -from dongtai_common.models.hook_type import HookType from dongtai_common.models.strategy import IastStrategyModel from dongtai_common.models.vulnerablity import IastVulnerabilityModel -from dongtai_common.models.project import IastProject from dongtai_common.utils import const from dongtai_conf import settings from dongtai_protocol.report.handler.report_handler_interface import IReportHandler @@ -131,5 +128,5 @@ def save(self): project_version_id=iast_vul.agent.project_version_id, pk__lt=iast_vul.id).delete() log_vul_found(iast_vul.agent.user_id, iast_vul.agent.bind_project.name, - iast_vul.agent.bind_project_id, iast_vul.id, + iast_vul.agent.bind_project_id, iast_vul.id, # type: ignore iast_vul.strategy.vul_name) diff --git a/dongtai_protocol/report/handler/narmal_vul_handler.py b/dongtai_protocol/report/handler/narmal_vul_handler.py index e1d115370..ca1bb463a 100644 --- a/dongtai_protocol/report/handler/narmal_vul_handler.py +++ b/dongtai_protocol/report/handler/narmal_vul_handler.py @@ -6,7 +6,6 @@ import json import logging -import random import time from dongtai_common.models.hook_type import HookType from dongtai_common.models.strategy import IastStrategyModel @@ -20,7 +19,7 @@ from dongtai_web.vul_log.vul_log import log_vul_found from dongtai_common.models.agent import IastAgent import re2 as re -from dongtai_common.models.header_vulnerablity import IastHeaderVulnerability, IastHeaderVulnerabilityDetail +from dongtai_common.models.header_vulnerablity import IastHeaderVulnerability from django.db import IntegrityError from dongtai_protocol import utils @@ -276,7 +275,7 @@ def save(self): ) log_vul_found(iast_vul.agent.user_id, iast_vul.agent.bind_project.name, - iast_vul.agent.bind_project_id, iast_vul.id, + iast_vul.agent.bind_project_id, iast_vul.id, # type: ignore iast_vul.strategy.vul_name) IastVulnerabilityModel.objects.filter( strategy_id=iast_vul.strategy_id, diff --git a/dongtai_protocol/views/agent_config.py b/dongtai_protocol/views/agent_config.py index 920867557..a5daaebb5 100644 --- a/dongtai_protocol/views/agent_config.py +++ b/dongtai_protocol/views/agent_config.py @@ -130,7 +130,7 @@ def get_agent_config(agent_id: int) -> Result: "enableAutoFallback": True, "performanceLimitRiskMaxMetricsCount": 30, } - interval_list = [] + interval_list: list[int] = [] for mg in MetricGroup: res = get_agent_config_by_scan(agent_id, mg) if isinstance(res, Err): diff --git a/dongtai_protocol/views/hook_profiles.py b/dongtai_protocol/views/hook_profiles.py index 3d94ffe98..265934c76 100644 --- a/dongtai_protocol/views/hook_profiles.py +++ b/dongtai_protocol/views/hook_profiles.py @@ -9,7 +9,6 @@ from dongtai_common.models.hook_strategy import HookStrategy from dongtai_common.models.hook_type import HookType from drf_spectacular.utils import extend_schema -from rest_framework.request import Request from dongtai_common.models.strategy import IastStrategyModel from dongtai_common.utils import const from dongtai_common.endpoint import OpenApiEndPoint, R @@ -100,7 +99,7 @@ def get_profiles(user=None, language_id=JAVA, full=False, system_only=False): "stack_blacklist": strategy.get("stack_blacklist"), }) strategy_details = sorted(strategy_details, - key=lambda item: item['value']) + key=lambda item: str(item['value'])) if not strategy_details: continue profiles.append({ diff --git a/dongtai_web/dongtai_sca/scan/tests.py b/dongtai_web/dongtai_sca/scan/tests.py index d1ab0417a..9b063b422 100644 --- a/dongtai_web/dongtai_sca/scan/tests.py +++ b/dongtai_web/dongtai_sca/scan/tests.py @@ -2,7 +2,6 @@ from .utils import get_nearest_version, get_latest_version from .utils import update_one_sca, new_update_one_sca from test.apiserver.test_agent_base import AgentTestCase -from typing import cast from .utils import get_package_vul, get_package from django.test import TestCase @@ -171,7 +170,8 @@ def test_update_one_sca_java_result_new(self): ).first() assert asset is not None # skip until sca data stable - self.assertGreaterEqual(cast(int, asset.vul_count), 0) + assert asset.vul_count is not None + self.assertGreaterEqual(asset.vul_count, 0) class SCAScanV2TestCase(AgentTestCase): diff --git a/dongtai_web/threshold/config_setting.py b/dongtai_web/threshold/config_setting.py index b17267006..81aa86344 100644 --- a/dongtai_web/threshold/config_setting.py +++ b/dongtai_web/threshold/config_setting.py @@ -8,7 +8,6 @@ from django.db.models import F from django.forms.models import model_to_dict from django.db.models import Max, Min -from functools import partial from inflection import underscore from collections.abc import Iterable from dongtai_common.models.agent_config import ( @@ -37,7 +36,7 @@ from dongtai_web.serializers.agent_config import AgentConfigSettingSerializer from rest_framework.serializers import ValidationError from rest_framework.utils.serializer_helpers import ReturnDict -from typing import Dict, cast +from typing import Dict from collections import OrderedDict _ResponseSerializer = get_response_serializer(status_msg_keypair=( @@ -332,7 +331,9 @@ def update(self, request, pk): def reset(self, request, pk): if IastCircuitConfig.objects.filter(pk=pk).exists(): config = IastCircuitConfig.objects.filter(pk=pk, ).first() - mg = MetricGroup(cast(int, config.metric_group)) + assert config is not None + assert config.metric_group is not None + mg = MetricGroup(config.metric_group) data = DEFAULT_CIRCUITCONFIG[mg.name] config_update(data, pk) return R.success() diff --git a/dongtai_web/views/agents.py b/dongtai_web/views/agents.py index abdf3dca3..f87902553 100644 --- a/dongtai_web/views/agents.py +++ b/dongtai_web/views/agents.py @@ -4,11 +4,9 @@ # software: PyCharm # project: lingzhi-webapi import logging -from django.db.models import Prefetch from dongtai_common.endpoint import UserEndPoint, R from django.forms.models import model_to_dict -from dongtai_common.utils import const from dongtai_web.serializers.agent import AgentSerializer from dongtai_web.utils import get_model_field from dongtai_common.models.agent import IastAgent @@ -17,7 +15,6 @@ from django.db.models import Q from django.utils.translation import gettext_lazy as _ from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer -from dongtai_web.base.paginator import ListPageMaker from django.core.cache import cache logger = logging.getLogger('dongtai-webapi') @@ -172,7 +169,7 @@ def get(self, request): ) end = [] for item in queryset: - one = model_to_dict(item) + one = model_to_dict(item) # type: ignore server_data = model_to_dict(item.server) one['cluster_name'] = server_data.get("cluster_name", "") one['cluster_version'] = server_data.get("cluster_version", "") diff --git a/dongtai_web/views/api_route_related_request.py b/dongtai_web/views/api_route_related_request.py index 0e7049874..b51da72f7 100644 --- a/dongtai_web/views/api_route_related_request.py +++ b/dongtai_web/views/api_route_related_request.py @@ -6,14 +6,13 @@ # @description : ###################################################################### -from dongtai_common.models.api_route import IastApiRoute, IastApiMethod, IastApiRoute, HttpMethod, IastApiResponse, IastApiMethodHttpMethodRelation, IastApiParameter +from dongtai_common.models.api_route import IastApiRoute from dongtai_common.models.agent_method_pool import MethodPool from dongtai_web.base.project_version import get_project_version, get_project_version_by_id from dongtai_common.endpoint import R, UserEndPoint from dongtai_common.models.agent import IastAgent from django.utils.translation import gettext_lazy as _ from django.db.models import Q -from django.forms.models import model_to_dict from dongtai_web.utils import sha1 from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer from rest_framework import serializers @@ -76,5 +75,7 @@ def get(self, request): q = q & Q( http_method__in=[_.method for _ in api_route.method.http_method.all()]) method = MethodPool.objects.filter(q).order_by('-update_time')[0:1].values() - data = list(method)[0] if method else {} - return R.success(data=data) + if method: + return R.success(data=list(method)[0]) + else: + return R.success(data={}) diff --git a/dongtai_web/views/api_route_search.py b/dongtai_web/views/api_route_search.py index 61de2a4ec..322489327 100644 --- a/dongtai_web/views/api_route_search.py +++ b/dongtai_web/views/api_route_search.py @@ -11,21 +11,12 @@ from dongtai_common.models.api_route import ( IastApiRoute, IastApiMethod, - IastApiRoute, HttpMethod, - IastApiResponse, - IastApiMethodHttpMethodRelation, - IastApiParameter, - FromWhereChoices, ) from dongtai_common.models.agent import IastAgent from dongtai_web.base.project_version import get_project_version, get_project_version_by_id from dongtai_common.models.vulnerablity import IastVulnerabilityModel -import hashlib -from dongtai_common.models.agent_method_pool import MethodPool from django.forms.models import model_to_dict -from dongtai_web.utils import checkcover, batch_queryset -from django.core.cache import caches from functools import partial from dongtai_common.models.hook_type import HookType from rest_framework import serializers @@ -293,7 +284,7 @@ def _get_vuls(uri, agents): uri=uri, agent_id__in=[_['id'] for _ in agents], is_del=0).values('hook_type_id', 'level_id', 'strategy_id', 'strategy__vul_name').distinct().all() - return [_get_hook_type(vul) for vul in vuls] + return [_get_hook_type(dict(vul)) for vul in vuls] def _get_hook_type(vul: dict) -> dict: diff --git a/pyproject.toml b/pyproject.toml index 37eb52305..42bb95481 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,4 +4,8 @@ disable = "C0330, C0326" [tool.pylint.format] max-line-length = "88" +[tool.mypy] +plugins = ["mypy_django_plugin.main"] +[tool.django-stubs] +django_settings_module = "dongtai_conf.settings" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 59fd13bd3..d27cd7b87 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,32 +5,32 @@ aliyun-python-sdk-kms==2.16.1 amqp==5.1.1 ; python_version >= '3.6' asgiref==3.7.2 async-timeout==4.0.2 ; python_version >= '3.6' -attrs==23.1.0 ; python_full_version >= '3.7.0' +attrs==23.1.0 ; python_version >= '3.7' autopep8==2.0.2 ; python_version >= '3.6' -billiard==4.1.0 ; python_full_version >= '3.7.0' +billiard==4.1.0 ; python_version >= '3.7' boto3==1.24.59 -boto3-stubs==1.26.144 +boto3-stubs==1.26.162 botocore==1.27.91 -botocore-stubs==1.29.144 +botocore-stubs==1.29.162 celery==5.3.0rc1 celery-singleton==0.3.1 certifi==2023.5.7 cffi==1.15.1 chardet==5.1.0 charset-normalizer==3.1.0 ; python_full_version >= '3.7.0' -click==8.1.3 ; python_full_version >= '3.7.0' -click-didyoumean==0.3.0 ; python_version < '4' and python_full_version >= '3.6.2' +click==8.1.3 ; python_version >= '3.7' +click-didyoumean==0.3.0 ; python_full_version >= '3.6.2' and python_full_version < '4.0.0' click-plugins==1.1.1 -click-repl==0.2.0 +click-repl==0.3.0 ; python_version >= '3.6' crcmod==1.7 cryptography==41.0.0 -dataclasses-json==0.5.7 +dataclasses-json==0.5.8 ddt==1.6.0 defusedxml==0.7.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' -diff-match-patch==20230430 ; python_full_version >= '3.7.0' +diff-match-patch==20230430 ; python_version >= '3.7' django==3.2.19 django-celery-beat==2.2.0 -django-cors-headers==4.0.0 +django-cors-headers==4.1.0 django-cprofile-middleware==1.0.5 django-elasticsearch-dsl==7.2.2 django-filter==23.2 @@ -44,11 +44,11 @@ django-seriously==0.4.0 django-silk==5.0.3 django-simple-captcha==0.5.17 django-stubs[compatible-mypy]==1.15.0 -django-stubs-ext==4.2.1 ; python_version >= '3.8' +django-stubs-ext==4.2.2 ; python_version >= '3.8' django-timezone-field==4.2.3 ; python_version >= '3.5' django-utils==0.0.2 django-utils-six==2.0 -django-xff==1.3.0 +django-xff==1.4.0 djangorestframework==3.12.4 djangorestframework-dataclasses==1.2.0 djangorestframework-stubs[compatible-mypy]==1.9.1 @@ -65,24 +65,24 @@ gunicorn==20.1.0 id-validator==1.0.20 idna==2.10 inflection==0.5.1 ; python_version >= '3.5' -jinja2==3.1.2 ; python_full_version >= '3.7.0' +jinja2==3.1.2 ; python_version >= '3.7' jmespath==0.10.0 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' jq==1.3.0 jsonlog==4.0.0 jsonschema==4.17.0 -kombu==5.3.0rc2 ; python_version >= '3.8' +kombu==5.3.1 ; python_version >= '3.8' lxml==4.9.1 marisa-trie==0.8.0 markuppy==1.14 -markupsafe==2.1.2 ; python_full_version >= '3.7.0' -marshmallow==3.19.0 ; python_full_version >= '3.7.0' +markupsafe==2.1.3 ; python_version >= '3.7' +marshmallow==3.19.0 ; python_version >= '3.7' marshmallow-enum==1.5.1 mock==5.0.2 ; python_version >= '3.6' -model-bakery==1.11.0 ; python_full_version >= '3.7.0' +model-bakery==1.12.0 ; python_version >= '3' mypy==1.0.1 mypy-extensions==1.0.0 ; python_version >= '3.5' -mysqlclient==2.1.1 -numpy==1.24.3 ; python_version >= '3.8' +mysqlclient==2.2.0 +numpy==1.25.0 ; python_version >= '3.9' odfpy==1.4.1 openpyxl==3.0.9 oss2==2.13.1 @@ -94,9 +94,9 @@ pycodestyle==2.10.0 ; python_version >= '3.6' pycparser==2.21 pycryptodome==3.18.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' pycryptodomex==3.14.1 -pydantic==1.10.8 ; python_full_version >= '3.7.0' +pydantic==1.10.9 ; python_version >= '3.7' pymysql==1.0.2 -pyparsing==3.0.9 ; python_full_version >= '3.6.8' +pyparsing==3.1.0 ; python_full_version >= '3.6.8' pyre2==0.3.6 pyrsistent==0.19.1 python-crontab==2.7.1 @@ -108,24 +108,24 @@ pyyaml==6.0 ; python_version >= '3.6' redis==4.4.4 requests==2.31.0 result==0.8.0 -s3transfer==0.6.1 ; python_full_version >= '3.7.0' +s3transfer==0.6.1 ; python_version >= '3.7' setuptools==65.5.1 simhash==2.1.2 six==1.15.0 sqlparse==0.4.4 ; python_version >= '3.5' -tablib[html,ods,xls,xlsx,yaml]==3.4.0 ; python_full_version >= '3.7.0' -tomli==2.0.1 ; python_full_version >= '3.7.0' -types-awscrt==0.16.19 ; python_version < '4' and python_full_version >= '3.7.0' +tablib[html,ods,xls,xlsx,yaml]==3.5.0 ; python_version >= '3.8' +tomli==2.0.1 ; python_version < '3.11' +types-awscrt==0.16.21 ; python_version >= '3.7' and python_version < '4.0' types-pymysql==1.0.19.7 -types-pyopenssl==23.2.0.0 +types-pyopenssl==23.2.0.1 types-python-dateutil==2.8.19.13 types-pytz==2023.3.0.0 types-pyyaml==6.0.12.10 -types-redis==4.5.5.2 +types-redis==4.6.0.0 types-requests==2.31.0.1 -types-s3transfer==0.6.1 ; python_version < '4' and python_full_version >= '3.7.0' +types-s3transfer==0.6.1 ; python_version >= '3.7' and python_version < '4.0' types-urllib3==1.26.25.13 -typing-extensions==4.6.2 +typing-extensions==4.6.3 typing-inspect==0.9.0 tzdata==2023.3 ; python_version >= '2' uritemplate==4.1.1 ; python_version >= '3.6' @@ -136,5 +136,5 @@ vine==5.0.0 ; python_version >= '3.6' wcwidth==0.2.6 xlrd==2.0.1 xlwt==1.3.0 -zope.event==4.6 -zope.interface==6.0 ; python_full_version >= '3.7.0' +zope.event==5.0 ; python_version >= '3.7' +zope.interface==6.0 ; python_version >= '3.7' From e9bd2f3326d3802f7629d897ea381085d136e9af Mon Sep 17 00:00:00 2001 From: st1020 Date: Wed, 28 Jun 2023 17:03:28 +0800 Subject: [PATCH 021/161] fix: type hint error --- dongtai_common/models/agent.py | 16 +++++------ dongtai_common/models/agent_config.py | 28 ++++++++----------- dongtai_common/models/agent_method_pool.py | 19 ++++++------- dongtai_common/models/api_route.py | 13 ++++----- dongtai_common/models/asset.py | 18 ++++++------ dongtai_common/models/asset_aggr.py | 12 ++++---- dongtai_common/models/asset_vul.py | 16 +++++------ dongtai_common/models/asset_vul_v2.py | 4 +-- dongtai_common/models/assetv2.py | 22 +++++++-------- dongtai_common/models/dast_integration.py | 4 +-- dongtai_common/models/department.py | 1 + dongtai_common/models/deploy.py | 4 +-- dongtai_common/models/document.py | 6 ++-- .../models/engine_monitoring_indicators.py | 4 +-- dongtai_common/models/header_vulnerablity.py | 2 +- dongtai_common/models/hook_strategy.py | 8 +++--- dongtai_common/models/hook_type.py | 5 ++-- dongtai_common/models/message.py | 6 ++-- dongtai_common/models/program_language.py | 2 +- dongtai_common/models/project.py | 16 ++++------- dongtai_common/models/project_report.py | 18 ++++++------ dongtai_common/models/project_version.py | 2 +- dongtai_common/models/recognize_rule.py | 2 +- dongtai_common/models/replay_method_pool.py | 14 +++++----- dongtai_common/models/sensitive_info.py | 2 +- dongtai_common/models/server.py | 8 +++--- dongtai_common/models/strategy.py | 6 ++-- dongtai_common/models/strategy_user.py | 2 +- dongtai_common/models/user.py | 6 ++-- dongtai_common/models/vul_level.py | 6 ++-- dongtai_common/models/vulnerablity.py | 5 ++-- dongtai_engine/tasks.py | 12 ++------ dongtai_web/dongtai_sca/models.py | 18 ++++++------ dongtai_web/threshold/config_setting.py | 7 +++-- 34 files changed, 145 insertions(+), 169 deletions(-) diff --git a/dongtai_common/models/agent.py b/dongtai_common/models/agent.py index 24dd5a55f..f9a0e4f4f 100644 --- a/dongtai_common/models/agent.py +++ b/dongtai_common/models/agent.py @@ -18,8 +18,8 @@ class IastAgent(models.Model): - token = models.CharField(max_length=255) - version = models.CharField(max_length=255) + token = models.CharField(max_length=255, blank=True) + version = models.CharField(max_length=255, blank=True) latest_time = models.IntegerField() user = models.ForeignKey(User, models.DO_NOTHING) server = models.ForeignKey( @@ -40,12 +40,12 @@ class IastAgent(models.Model): project_version = models.ForeignKey(IastProjectVersion, on_delete=models.CASCADE, default=-1) - project_name = models.CharField(max_length=255) + project_name = models.CharField(max_length=255, blank=True) online = models.PositiveSmallIntegerField(default=0) - language = models.CharField(max_length=10) - filepathsimhash = models.CharField(max_length=255, blank=True, default='') - servicetype = models.CharField(max_length=255, blank=True, default='') - alias = models.CharField(default='', max_length=255, blank=True) + language = models.CharField(max_length=10, blank=True) + filepathsimhash = models.CharField(max_length=255, blank=True) + servicetype = models.CharField(max_length=255, blank=True) + alias = models.CharField(max_length=255, blank=True) startup_time = models.IntegerField(default=0) register_time = models.IntegerField(default=0) actual_running_status = models.IntegerField(default=1) @@ -91,7 +91,7 @@ class IastAgentEvent(models.Model): agent = models.ForeignKey(IastAgent, on_delete=models.CASCADE, related_name='new_events') - name = models.CharField(default='', max_length=255) + name = models.CharField(max_length=255, blank=True) time = models.IntegerField(default=lambda: int(time()), blank=True, null=True) diff --git a/dongtai_common/models/agent_config.py b/dongtai_common/models/agent_config.py index 085aef5e0..1aa3736fd 100644 --- a/dongtai_common/models/agent_config.py +++ b/dongtai_common/models/agent_config.py @@ -10,11 +10,11 @@ class IastAgentConfig(models.Model): user = models.ForeignKey(User, models.DO_NOTHING) details = models.JSONField() - hostname = models.CharField(max_length=255, ) - ip = models.CharField(max_length=100, ) + hostname = models.CharField(max_length=255, blank=True) + ip = models.CharField(max_length=100, blank=True) port = models.IntegerField() - cluster_name = models.CharField(max_length=255, ) - cluster_version = models.CharField(max_length=100, ) + cluster_name = models.CharField(max_length=255, blank=True) + cluster_version = models.CharField(max_length=100, blank=True) priority = models.IntegerField() create_time = models.IntegerField() @@ -79,7 +79,6 @@ class MetricType(IntegerChoices): 4: "%", 5: "kb", 6: "个", - 6: "个", 7: "个", 8: "个", 9: "次", @@ -112,25 +111,20 @@ class ApplicationMetricType(IntegerChoices): class IastCircuitConfig(models.Model): user = models.ForeignKey(User, models.DO_NOTHING) - name = models.CharField(max_length=200, ) - metric_types = models.CharField(max_length=2000, ) + name = models.CharField(max_length=200, blank=True) + metric_types = models.CharField(max_length=2000, blank=True) target_types = models.CharField(max_length=2000, blank=True, - null=True, db_column='targets') system_type = models.IntegerField(blank=True, null=True) is_enable = models.IntegerField() - is_deleted = models.IntegerField(default=0, ) + is_deleted = models.IntegerField(default=0) deal = models.IntegerField() interval = models.IntegerField(default=30) metric_group = models.IntegerField() priority = models.IntegerField() - create_time = models.IntegerField(blank=True, - null=True, - default=lambda: int(time())) - update_time = models.IntegerField(blank=True, - null=True, - default=lambda: int(time())) + create_time = models.IntegerField(blank=True, default=lambda: int(time())) + update_time = models.IntegerField(blank=True, default=lambda: int(time())) class Meta: managed = get_managed() @@ -142,7 +136,7 @@ class IastCircuitTarget(models.Model): on_delete=models.CASCADE) target_type = models.IntegerField() opt = models.IntegerField() - value = models.CharField(max_length=200, ) + value = models.CharField(max_length=200, blank=True) class Meta: managed = get_managed() @@ -154,7 +148,7 @@ class IastCircuitMetric(models.Model): on_delete=models.CASCADE) metric_type = models.IntegerField() opt = models.IntegerField() - value = models.CharField(max_length=200, ) + value = models.CharField(max_length=200, blank=True) class Meta: managed = get_managed() diff --git a/dongtai_common/models/agent_method_pool.py b/dongtai_common/models/agent_method_pool.py index c9cd73027..fbfac3f9f 100644 --- a/dongtai_common/models/agent_method_pool.py +++ b/dongtai_common/models/agent_method_pool.py @@ -21,14 +21,12 @@ class MethodPool(models.Model): id = models.BigAutoField(primary_key=True) agent = models.ForeignKey(IastAgent, models.DO_NOTHING, - blank=True, - null=True, db_constraint=False) - url = models.CharField(max_length=2000) - uri = models.CharField(max_length=2000) - http_method = models.CharField(max_length=10, default='') - http_scheme = models.CharField(max_length=20) - http_protocol = models.CharField(max_length=255) + url = models.CharField(max_length=2000, blank=True) + uri = models.CharField(max_length=2000, blank=True) + http_method = models.CharField(max_length=10, blank=True) + http_scheme = models.CharField(max_length=20, blank=True) + http_protocol = models.CharField(max_length=255, blank=True) req_header = models.CharField(max_length=2000, blank=True, null=True) req_params = models.CharField(max_length=2000, blank=True, null=True) req_data = models.CharField(max_length=4000, blank=True, null=True) @@ -37,13 +35,12 @@ class MethodPool(models.Model): req_header_fs = models.TextField(db_column='req_header_for_search') context_path = models.CharField(max_length=255, blank=True, null=True) method_pool = models.TextField() # This field type is a guess. - pool_sign = models.CharField(unique=True, - max_length=40) # This field type is a guess. - clent_ip = models.CharField(max_length=255) + pool_sign = models.CharField(unique=True, blank=True, max_length=40) # This field type is a guess. + clent_ip = models.CharField(max_length=255, blank=True) create_time = models.IntegerField() update_time = models.IntegerField() uri_sha1 = models.CharField(max_length=40, - default='', + blank=True, db_index=True) sinks = models.ManyToManyField( HookStrategy, diff --git a/dongtai_common/models/api_route.py b/dongtai_common/models/api_route.py index e21df5ebb..fd6d8e356 100644 --- a/dongtai_common/models/api_route.py +++ b/dongtai_common/models/api_route.py @@ -14,7 +14,7 @@ class HttpMethod(models.Model): - method = models.CharField(max_length=100) + method = models.CharField(max_length=100, blank=True) class Meta: managed = get_managed() @@ -22,7 +22,7 @@ class Meta: class IastApiMethod(models.Model): - method = models.CharField(max_length=100) + method = models.CharField(max_length=100, blank=True) http_method = models.ManyToManyField( HttpMethod, through='IastApiMethodHttpMethodRelation') @@ -89,11 +89,8 @@ class Meta: class IastApiParameter(models.Model): - name = models.CharField(max_length=100) - parameter_type = models.CharField(max_length=100, - blank=True, - default='', - db_column='type') + name = models.CharField(max_length=100, blank=True) + parameter_type = models.CharField(max_length=100, blank=True, db_column='type') annotation = models.CharField(max_length=500, blank=True) route = models.ForeignKey(IastApiRoute, on_delete=models.CASCADE, @@ -108,7 +105,7 @@ class Meta: class IastApiResponse(models.Model): - return_type = models.CharField(max_length=100) + return_type = models.CharField(max_length=100, blank=True) route = models.ForeignKey(IastApiRoute, on_delete=models.CASCADE, db_constraint=False, diff --git a/dongtai_common/models/asset.py b/dongtai_common/models/asset.py index 7759147a4..9706658e1 100644 --- a/dongtai_common/models/asset.py +++ b/dongtai_common/models/asset.py @@ -26,14 +26,14 @@ class Asset(models.Model): id = models.BigAutoField(primary_key=True) - package_name = models.CharField(max_length=255) - package_path = models.CharField(max_length=255) - signature_algorithm = models.CharField(max_length=255) - signature_value = models.CharField(max_length=255) + package_name = models.CharField(max_length=255, blank=True) + package_path = models.CharField(max_length=255, blank=True) + signature_algorithm = models.CharField(max_length=255, blank=True) + signature_value = models.CharField(max_length=255, blank=True) dt = models.IntegerField() version = models.CharField(max_length=255, blank=True) - safe_version = models.CharField(max_length=255, blank=True, default='') - last_version = models.CharField(max_length=255, blank=True, default='') + safe_version = models.CharField(max_length=255, blank=True) + last_version = models.CharField(max_length=255, blank=True) level = models.ForeignKey(IastVulLevel, models.DO_NOTHING, default=4) vul_count = models.IntegerField() vul_critical_count = models.IntegerField(default=0) @@ -53,9 +53,9 @@ class Asset(models.Model): project_version = models.ForeignKey(IastProjectVersion, on_delete=models.CASCADE, default=-1) user = models.ForeignKey(User, models.DO_NOTHING, default=-1) - project_name = models.CharField(max_length=255, default='') - language = models.CharField(max_length=32, default='') - license = models.CharField(max_length=64, default='') + project_name = models.CharField(max_length=255, blank=True) + language = models.CharField(max_length=32, blank=True) + license = models.CharField(max_length=64, blank=True) dependency_level = models.IntegerField(default=0) parent_dependency_id = models.IntegerField(default=0) is_del = models.SmallIntegerField(default=0) diff --git a/dongtai_common/models/asset_aggr.py b/dongtai_common/models/asset_aggr.py index aa0c6b6ab..030e70189 100644 --- a/dongtai_common/models/asset_aggr.py +++ b/dongtai_common/models/asset_aggr.py @@ -13,11 +13,11 @@ class AssetAggr(models.Model): - package_name = models.CharField(max_length=255) + package_name = models.CharField(max_length=255, blank=True) signature_value = models.CharField(max_length=255, blank=True) - version = models.CharField(max_length=255) - safe_version = models.CharField(max_length=255, blank=True, default='') - last_version = models.CharField(max_length=255, blank=True, default='') + version = models.CharField(max_length=255, blank=True) + safe_version = models.CharField(max_length=255, blank=True) + last_version = models.CharField(max_length=255, blank=True) level = models.ForeignKey(IastVulLevel, models.DO_NOTHING) vul_count = models.IntegerField() vul_critical_count = models.IntegerField(default=0) @@ -26,8 +26,8 @@ class AssetAggr(models.Model): vul_low_count = models.IntegerField(default=0) vul_info_count = models.IntegerField(default=0) project_count = models.IntegerField(blank=True) - language = models.CharField(max_length=32, default='') - license = models.CharField(max_length=64, blank=True, default='') + language = models.CharField(max_length=32, blank=True) + license = models.CharField(max_length=64, blank=True) is_del = models.SmallIntegerField(default=0) class Meta: diff --git a/dongtai_common/models/asset_vul.py b/dongtai_common/models/asset_vul.py index 1b912fc3e..293c98b43 100644 --- a/dongtai_common/models/asset_vul.py +++ b/dongtai_common/models/asset_vul.py @@ -13,22 +13,22 @@ class IastAssetVul(models.Model): - vul_name = models.CharField(max_length=255) + vul_name = models.CharField(max_length=255, blank=True) vul_detail = models.TextField() # 漏洞类型等级 level = models.ForeignKey(IastVulLevel, models.DO_NOTHING) license = models.CharField(max_length=50, blank=True, null=True) # 开源许可证 风险等级 # 1 高 2中 3低 0无风险 license_level = models.SmallIntegerField(blank=True, null=True) - aql = models.CharField(max_length=100) - package_name = models.CharField(max_length=100) + aql = models.CharField(max_length=100, blank=True) + package_name = models.CharField(max_length=100, blank=True) package_hash = models.CharField(max_length=100, blank=True, null=True) - package_version = models.CharField(max_length=50) + package_version = models.CharField(max_length=50, blank=True) package_safe_version = models.CharField(max_length=50, blank=True, null=True) package_latest_version = models.CharField(max_length=50, blank=True, null=True) - package_language = models.CharField(max_length=10) + package_language = models.CharField(max_length=10, blank=True) vul_cve_nums = models.JSONField() - vul_serial = models.CharField(max_length=100) # 漏洞编号 CWE|CVE等数据 + vul_serial = models.CharField(max_length=100, blank=True) # 漏洞编号 CWE|CVE等数据 have_article = models.SmallIntegerField() have_poc = models.SmallIntegerField() cve_code = models.CharField(max_length=64, blank=True, null=True) @@ -88,8 +88,8 @@ class Meta: class IastAssetVulType(models.Model): - cwe_id = models.CharField(max_length=20, blank=True, default='') - name = models.CharField(max_length=100, blank=True, default='') + cwe_id = models.CharField(max_length=20, blank=True) + name = models.CharField(max_length=100, blank=True) class Meta: managed = get_managed() diff --git a/dongtai_common/models/asset_vul_v2.py b/dongtai_common/models/asset_vul_v2.py index 8fb0e1583..53dbeb49f 100644 --- a/dongtai_common/models/asset_vul_v2.py +++ b/dongtai_common/models/asset_vul_v2.py @@ -4,7 +4,7 @@ class IastAssetVulV2(models.Model): - vul_name = models.CharField(max_length=255) + vul_name = models.CharField(max_length=255, blank=True) vul_detail = models.TextField() # 漏洞类型等级 level = models.IntegerField(choices=AssetRiskLevel.choices, @@ -16,7 +16,7 @@ class IastAssetVulV2(models.Model): references = models.JSONField(default=list) change_time = models.IntegerField() published_time = models.IntegerField() - vul_id = models.CharField(max_length=255, unique=True) + vul_id = models.CharField(max_length=255, unique=True, blank=True) vul_type = models.JSONField() vul_codes = models.JSONField() affected_versions = models.JSONField() diff --git a/dongtai_common/models/assetv2.py b/dongtai_common/models/assetv2.py index c6227e493..99b442c5e 100644 --- a/dongtai_common/models/assetv2.py +++ b/dongtai_common/models/assetv2.py @@ -26,12 +26,12 @@ class AssetRiskLevel(IntegerChoices): class AssetV2(models.Model): id = models.BigAutoField(primary_key=True) - package_name = models.CharField(max_length=255, unique=True) - package_path = models.CharField(max_length=255) - signature_algorithm = models.CharField(max_length=255) - signature_value = models.CharField(max_length=255) + package_name = models.CharField(max_length=255, unique=True, blank=True) + package_path = models.CharField(max_length=255, blank=True) + signature_algorithm = models.CharField(max_length=255, blank=True) + signature_value = models.CharField(max_length=255, blank=True) dt = models.IntegerField(blank=True, default=lambda: int(time.time())) - version = models.CharField(max_length=255) + version = models.CharField(max_length=255, blank=True) project = models.ForeignKey(IastProject, on_delete=models.CASCADE, blank=True, @@ -49,7 +49,7 @@ class AssetV2(models.Model): #is_reconized = models.IntegerField(blank=True, null=True) aql = models.ForeignKey('AssetV2Global', to_field='aql', - default='', + blank=True, db_column="aql", on_delete=models.DO_NOTHING) @@ -60,7 +60,7 @@ class Meta: class AssetV2Global(models.Model): id = models.BigAutoField(primary_key=True) - package_name = models.CharField(max_length=255) + package_name = models.CharField(max_length=255, blank=True) package_fullname = models.ForeignKey( 'IastPackageGAInfo', on_delete=models.DO_NOTHING, @@ -68,9 +68,9 @@ class AssetV2Global(models.Model): db_column="package_fullname", to_field="package_fullname", ) - signature_algorithm = models.CharField(max_length=255) - signature_value = models.CharField(max_length=255) - version = models.CharField(max_length=255) + signature_algorithm = models.CharField(max_length=255, blank=True) + signature_value = models.CharField(max_length=255, blank=True) + version = models.CharField(max_length=255, blank=True) level = models.IntegerField( choices=AssetRiskLevel.choices, blank=True, @@ -129,7 +129,7 @@ class Meta: class IastPackageGAInfo(models.Model): - package_fullname = models.CharField(max_length=255, unique=True) + package_fullname = models.CharField(max_length=255, unique=True, blank=True) affected_versions = models.JSONField(blank=True, default=list) unaffected_versions = models.JSONField(blank=True, default=list) diff --git a/dongtai_common/models/dast_integration.py b/dongtai_common/models/dast_integration.py index c29c2796f..ca2254d11 100644 --- a/dongtai_common/models/dast_integration.py +++ b/dongtai_common/models/dast_integration.py @@ -9,9 +9,9 @@ class IastDastIntegration(models.Model): vul_name = models.CharField(max_length=255, blank=True) - detail = models.TextField(blank=True, default='') + detail = models.TextField(blank=True) vul_level = models.ForeignKey(IastVulLevel, models.DO_NOTHING, blank=True) - payload = models.CharField(max_length=255, blank=True, default='') + payload = models.CharField(max_length=255, blank=True) target = models.CharField(max_length=255, blank=True) vul_type = models.CharField(max_length=255, blank=True) dast_tag = models.CharField(max_length=255, blank=True) diff --git a/dongtai_common/models/department.py b/dongtai_common/models/department.py index f9e0b9188..8f02922cb 100644 --- a/dongtai_common/models/department.py +++ b/dongtai_common/models/department.py @@ -44,6 +44,7 @@ class Department(PermissionsMixin): name = models.CharField( _('name'), unique=True, + blank=True, max_length=100, error_messages={ 'unique': _("A department with that department name already exists."), diff --git a/dongtai_common/models/deploy.py b/dongtai_common/models/deploy.py index f306852a0..69f1a1448 100644 --- a/dongtai_common/models/deploy.py +++ b/dongtai_common/models/deploy.py @@ -11,8 +11,8 @@ class IastDeployDesc(models.Model): desc = models.TextField() - middleware = models.CharField(max_length=255) - language = models.CharField(max_length=255) + middleware = models.CharField(max_length=255, blank=True) + language = models.CharField(max_length=255, blank=True) class Meta: managed = get_managed() diff --git a/dongtai_common/models/document.py b/dongtai_common/models/document.py index ca4193aec..d23f95b1a 100644 --- a/dongtai_common/models/document.py +++ b/dongtai_common/models/document.py @@ -3,9 +3,9 @@ class IastDocument(models.Model): - title = models.CharField(max_length=100) - url = models.CharField(max_length=2000) - language = models.CharField(max_length=100) + title = models.CharField(max_length=100, blank=True) + url = models.CharField(max_length=2000, blank=True) + language = models.CharField(max_length=100, blank=True) weight = models.IntegerField(default=0) class Meta: diff --git a/dongtai_common/models/engine_monitoring_indicators.py b/dongtai_common/models/engine_monitoring_indicators.py index 5806354d5..cd2d8892e 100644 --- a/dongtai_common/models/engine_monitoring_indicators.py +++ b/dongtai_common/models/engine_monitoring_indicators.py @@ -11,8 +11,8 @@ class IastEnginMonitoringIndicators(models.Model): - key = models.CharField(max_length=100, default='', unique=True) - name = models.CharField(max_length=100, default='') + key = models.CharField(max_length=100, blank=True, unique=True) + name = models.CharField(max_length=100, blank=True) class Meta: managed = get_managed() diff --git a/dongtai_common/models/header_vulnerablity.py b/dongtai_common/models/header_vulnerablity.py index 677169b66..8ddea9bb0 100644 --- a/dongtai_common/models/header_vulnerablity.py +++ b/dongtai_common/models/header_vulnerablity.py @@ -17,7 +17,7 @@ class IastHeaderVulnerability(models.Model): on_delete=models.DO_NOTHING, default=-1, db_constraint=False) - url = models.CharField(max_length=255, default='') + url = models.CharField(max_length=255, blank=True) vul = models.ForeignKey(IastVulnerabilityModel, on_delete=models.DO_NOTHING, default=-1, diff --git a/dongtai_common/models/hook_strategy.py b/dongtai_common/models/hook_strategy.py index bf87f6803..6bcf639ee 100644 --- a/dongtai_common/models/hook_strategy.py +++ b/dongtai_common/models/hook_strategy.py @@ -31,17 +31,17 @@ class HookStrategy(models.Model): - value = models.CharField(max_length=255) + value = models.CharField(max_length=255, blank=True) source = models.CharField(max_length=255, blank=True) target = models.CharField(max_length=255, blank=True) - inherit = models.CharField(max_length=255) + inherit = models.CharField(max_length=255, blank=True) track = models.CharField(max_length=5, blank=True) create_time = models.IntegerField(default=lambda: int(time())) update_time = models.IntegerField(default=lambda: int(time())) created_by = models.IntegerField() enable = models.IntegerField(default=1) language = models.ForeignKey(IastProgramLanguage, - default='', + blank=True, on_delete=models.DO_NOTHING, db_constraint=False) type = models.IntegerField() @@ -73,7 +73,7 @@ class HookStrategy(models.Model): tags = models.JSONField(default=list) untags = models.JSONField(default=list) stack_blacklist = models.JSONField(default=list) - command = models.CharField(max_length=128, default="") + command = models.CharField(max_length=128, blank=True) class Meta: managed = get_managed() diff --git a/dongtai_common/models/hook_type.py b/dongtai_common/models/hook_type.py index 5e0c4ed16..23f297d6a 100644 --- a/dongtai_common/models/hook_type.py +++ b/dongtai_common/models/hook_type.py @@ -12,15 +12,14 @@ class HookType(models.Model): type = models.IntegerField() - name = models.CharField(max_length=255) - value = models.CharField(max_length=255) + name = models.CharField(max_length=255, blank=True) + value = models.CharField(max_length=255, blank=True) enable = models.IntegerField(blank=True) create_time = models.IntegerField(blank=True, null=True, default=lambda: int(time())) update_time = models.IntegerField(blank=True, null=True, default=lambda: int(time())) created_by = models.IntegerField(blank=True) language = models.ForeignKey(IastProgramLanguage, blank=True, - default='', on_delete=models.DO_NOTHING, db_constraint=False) vul_strategy = models.ForeignKey( diff --git a/dongtai_common/models/message.py b/dongtai_common/models/message.py index c58755611..010d77ee6 100644 --- a/dongtai_common/models/message.py +++ b/dongtai_common/models/message.py @@ -13,7 +13,7 @@ class IastMessageType(models.Model): - name = models.CharField(max_length=100, default='') + name = models.CharField(max_length=100, blank=True) class Meta: managed = get_managed() @@ -21,8 +21,8 @@ class Meta: class IastMessage(models.Model): - message = models.CharField(max_length=512, default='') - relative_url = models.CharField(max_length=512, default='') + message = models.CharField(max_length=512, blank=True) + relative_url = models.CharField(max_length=512, blank=True) create_time = models.IntegerField(default=lambda: int(time())) read_time = models.IntegerField(default=0) is_read = models.IntegerField(default=0) diff --git a/dongtai_common/models/program_language.py b/dongtai_common/models/program_language.py index 86b5546e3..0e55f2beb 100644 --- a/dongtai_common/models/program_language.py +++ b/dongtai_common/models/program_language.py @@ -12,7 +12,7 @@ class IastProgramLanguage(models.Model): - name = models.CharField(max_length=255) + name = models.CharField(max_length=255, blank=True) class Meta: managed = get_managed() diff --git a/dongtai_common/models/project.py b/dongtai_common/models/project.py index 0a739249f..9ce04a185 100644 --- a/dongtai_common/models/project.py +++ b/dongtai_common/models/project.py @@ -48,8 +48,8 @@ def to_full_project_args(self): } class IastProject(models.Model): - name = models.CharField(max_length=255) - mode = models.CharField(default="插桩模式", max_length=255) + name = models.CharField(max_length=255, blank=True) + mode = models.CharField(default="插桩模式", max_length=255, blank=True) vul_count = models.PositiveIntegerField(blank=True, null=True) agent_count = models.IntegerField(blank=True, null=True) latest_time = models.IntegerField(default=lambda: int(time.time())) @@ -61,20 +61,16 @@ class IastProject(models.Model): null=True) vul_validation = models.IntegerField(default=0, choices=VulValidation.choices) - base_url = models.CharField(max_length=255, blank=True, default='') - test_req_header_key = models.CharField(max_length=511, - blank=True, - default='') - test_req_header_value = models.CharField(max_length=511, - blank=True, - default='') + base_url = models.CharField(max_length=255, blank=True) + test_req_header_key = models.CharField(max_length=511, blank=True) + test_req_header_value = models.CharField(max_length=511, blank=True) data_gather = models.JSONField(null=True) data_gather_is_followglobal = models.IntegerField(default=1) blacklist_is_followglobal = models.IntegerField(default=1) department = models.ForeignKey(Department, models.DO_NOTHING) template = models.ForeignKey(IastProjectTemplate, models.DO_NOTHING) enable_log = models.BooleanField(null=True) - log_level = models.CharField(max_length=511, null=True) + log_level = models.CharField(max_length=511, null=True, blank=True) class Meta: managed = get_managed() diff --git a/dongtai_common/models/project_report.py b/dongtai_common/models/project_report.py index 3a3e8a27e..8baf3f6cc 100644 --- a/dongtai_common/models/project_report.py +++ b/dongtai_common/models/project_report.py @@ -19,20 +19,20 @@ class ProjectReport(models.Model): user = models.ForeignKey(User, models.DO_NOTHING) project = models.ForeignKey(IastProject, models.DO_NOTHING) - type = models.CharField(max_length=10) - language = models.CharField(max_length=10) + type = models.CharField(max_length=10, blank=True) + language = models.CharField(max_length=10, blank=True) status = models.IntegerField(default=0) - path = models.CharField(default='', max_length=255, blank=True) + path = models.CharField(max_length=255, blank=True) file = models.BinaryField(blank=True, null=True) create_time = models.IntegerField(default=0) is_del = models.SmallIntegerField(default=0) - level_png = models.CharField(default='', max_length=255, blank=True, null=True) - trend_png = models.CharField(default='', max_length=255, blank=True, null=True) - version_str = models.CharField(default='', max_length=255, blank=True, null=True) - vul_type_str = models.CharField(default='', max_length=255, blank=True, null=True) - sca_type_str = models.TextField(default='', blank=True, null=True) + level_png = models.CharField(max_length=255, blank=True, null=True) + trend_png = models.CharField(max_length=255, blank=True, null=True) + version_str = models.CharField(max_length=255, blank=True, null=True) + vul_type_str = models.CharField(max_length=255, blank=True, null=True) + sca_type_str = models.TextField(blank=True, null=True) vul_id = models.IntegerField(default=0) - report_name = models.CharField(default='', max_length=255, blank=True, null=True) + report_name = models.CharField(max_length=255, blank=True, null=True) class Meta: managed = get_managed() diff --git a/dongtai_common/models/project_version.py b/dongtai_common/models/project_version.py index 00ea1e0fa..50c6901d0 100644 --- a/dongtai_common/models/project_version.py +++ b/dongtai_common/models/project_version.py @@ -13,7 +13,7 @@ class IastProjectVersion(models.Model): - version_name = models.CharField(max_length=255) + version_name = models.CharField(max_length=255, blank=True) description = models.TextField(blank=True) current_version = models.PositiveSmallIntegerField(default=0) status = models.PositiveSmallIntegerField() diff --git a/dongtai_common/models/recognize_rule.py b/dongtai_common/models/recognize_rule.py index 183dba58d..43b90029c 100644 --- a/dongtai_common/models/recognize_rule.py +++ b/dongtai_common/models/recognize_rule.py @@ -15,7 +15,7 @@ class IastRecognizeRule(models.Model): null=True, default=-1, db_constraint=False) - rule_detail = models.CharField(max_length=255, default='', blank=True) + rule_detail = models.CharField(max_length=255, blank=True) rule_type = models.IntegerField(choices=RuleTypeChoices.choices) created = models.DateTimeField(auto_now_add=True) updated = models.DateTimeField(auto_now=True) diff --git a/dongtai_common/models/replay_method_pool.py b/dongtai_common/models/replay_method_pool.py index bcd4b7b63..29b90a31d 100644 --- a/dongtai_common/models/replay_method_pool.py +++ b/dongtai_common/models/replay_method_pool.py @@ -13,19 +13,19 @@ class IastAgentMethodPoolReplay(models.Model): id = models.BigAutoField(primary_key=True) agent = models.ForeignKey(IastAgent, models.DO_NOTHING) - url = models.CharField(max_length=2000) - uri = models.CharField(max_length=2000) - http_method = models.CharField(max_length=10) - http_scheme = models.CharField(max_length=20) - http_protocol = models.CharField(max_length=255) - req_header = models.CharField(max_length=2000) + url = models.CharField(max_length=2000, blank=True) + uri = models.CharField(max_length=2000, blank=True) + http_method = models.CharField(max_length=10, blank=True) + http_scheme = models.CharField(max_length=20, blank=True) + http_protocol = models.CharField(max_length=255, blank=True) + req_header = models.CharField(max_length=2000, blank=True) req_params = models.CharField(max_length=2000, blank=True, null=True) req_data = models.CharField(max_length=4000, blank=True, null=True) res_header = models.CharField(max_length=1000, blank=True, null=True) res_body = models.CharField(max_length=1000, blank=True, null=True) context_path = models.CharField(max_length=255, blank=True, null=True) method_pool = models.TextField() # This field type is a guess. - clent_ip = models.CharField(max_length=255) + clent_ip = models.CharField(max_length=255, blank=True) create_time = models.IntegerField() update_time = models.IntegerField() replay_id = models.IntegerField() diff --git a/dongtai_common/models/sensitive_info.py b/dongtai_common/models/sensitive_info.py index 68fbf8c4d..45116d338 100644 --- a/dongtai_common/models/sensitive_info.py +++ b/dongtai_common/models/sensitive_info.py @@ -13,7 +13,7 @@ class IastPatternType(models.Model): - name = models.CharField(max_length=255) + name = models.CharField(max_length=255, blank=True) id = models.IntegerField(default=0, db_column='value') logi_id = models.BigAutoField(primary_key=True, db_column='id') diff --git a/dongtai_common/models/server.py b/dongtai_common/models/server.py index 2d0468f41..78a5af039 100644 --- a/dongtai_common/models/server.py +++ b/dongtai_common/models/server.py @@ -9,23 +9,23 @@ class IastServer(models.Model): - hostname = models.CharField(max_length=255) + hostname = models.CharField(max_length=255, blank=True) ip = models.CharField(max_length=255, blank=True) port = models.IntegerField() environment = models.TextField(blank=True, null=True) path = models.CharField(max_length=255, blank=True) - status = models.CharField(max_length=255) + status = models.CharField(max_length=255, blank=True) container = models.CharField(max_length=255, blank=True) container_path = models.CharField(max_length=255, blank=True) cluster_name = models.CharField(max_length=255, blank=True) cluster_version = models.CharField(max_length=100, blank=True) command = models.TextField(blank=True) - env = models.CharField(max_length=255) + env = models.CharField(max_length=255, blank=True) runtime = models.CharField(max_length=255, blank=True) create_time = models.IntegerField() update_time = models.IntegerField() network = models.CharField(max_length=255, blank=True) - protocol = models.CharField(max_length=255, blank=True, default='') + protocol = models.CharField(max_length=255, blank=True) pid = models.IntegerField(blank=True) ipaddresslist = models.JSONField(null=False, default=list) diff --git a/dongtai_common/models/strategy.py b/dongtai_common/models/strategy.py index a16203c63..6f11a1293 100644 --- a/dongtai_common/models/strategy.py +++ b/dongtai_common/models/strategy.py @@ -9,11 +9,11 @@ class IastStrategyModel(models.Model): user = models.ForeignKey(User, models.DO_NOTHING) - vul_type = models.CharField(max_length=255) + vul_type = models.CharField(max_length=255, blank=True) level = models.ForeignKey(IastVulLevel, models.DO_NOTHING) - state = models.CharField(max_length=255) + state = models.CharField(max_length=255, blank=True) dt = models.IntegerField(blank=True, default=lambda: int(time())) - vul_name = models.CharField(max_length=255) + vul_name = models.CharField(max_length=255, blank=True) vul_desc = models.TextField() vul_fix = models.TextField(blank=True) hook_type = models.ForeignKey(HookType, diff --git a/dongtai_common/models/strategy_user.py b/dongtai_common/models/strategy_user.py index 98781007b..cae6a51a8 100644 --- a/dongtai_common/models/strategy_user.py +++ b/dongtai_common/models/strategy_user.py @@ -7,7 +7,7 @@ class IastStrategyUser(models.Model): id = models.BigAutoField(primary_key=True) - name = models.CharField(max_length=200) + name = models.CharField(max_length=200, blank=True) content = models.TextField(blank=True) user = models.ForeignKey(User, models.DO_NOTHING) status = models.IntegerField(blank=True) diff --git a/dongtai_common/models/user.py b/dongtai_common/models/user.py index 8499d9efa..5043ab908 100644 --- a/dongtai_common/models/user.py +++ b/dongtai_common/models/user.py @@ -69,9 +69,9 @@ def create_system_user(self, class User(AbstractUser, PermissionsMixin): - is_superuser: int = models.IntegerField(default=0) - phone = models.CharField(max_length=15) - default_language = models.CharField(max_length=15) + is_superuser = models.IntegerField(default=0) + phone = models.CharField(max_length=15, blank=True) + default_language = models.CharField(max_length=15, blank=True) objects = SaaSUserManager() using_department = None diff --git a/dongtai_common/models/vul_level.py b/dongtai_common/models/vul_level.py index 91881acae..3486edbca 100644 --- a/dongtai_common/models/vul_level.py +++ b/dongtai_common/models/vul_level.py @@ -9,9 +9,9 @@ class IastVulLevel(models.Model): - name = models.CharField(max_length=255) - name_value = models.CharField(max_length=255) - name_type = models.CharField(max_length=255) + name = models.CharField(max_length=255, blank=True) + name_value = models.CharField(max_length=255, blank=True) + name_type = models.CharField(max_length=255, blank=True) class Meta: managed = get_managed() diff --git a/dongtai_common/models/vulnerablity.py b/dongtai_common/models/vulnerablity.py index 3af30ab1c..4133ace9f 100644 --- a/dongtai_common/models/vulnerablity.py +++ b/dongtai_common/models/vulnerablity.py @@ -16,7 +16,7 @@ from dongtai_common.models.project_version import IastProjectVersion class IastVulnerabilityStatus(models.Model): - name = models.CharField(max_length=100, blank=True, default='') + name = models.CharField(max_length=100, blank=True) class Meta: managed = get_managed() @@ -64,8 +64,7 @@ class IastVulnerabilityModel(models.Model): client_ip = models.CharField(max_length=255, blank=True) param_name = models.CharField(max_length=255, blank=True, - null=True, - default='') + null=True) is_del = models.SmallIntegerField(blank=True, default=0) method_pool_id = models.IntegerField(default=-1, blank=True) strategy = models.ForeignKey(IastStrategyModel, diff --git a/dongtai_engine/tasks.py b/dongtai_engine/tasks.py index 451693bfd..5b4d6f3a6 100644 --- a/dongtai_engine/tasks.py +++ b/dongtai_engine/tasks.py @@ -9,13 +9,11 @@ import hashlib import json import time -import random from json import JSONDecodeError from celery import shared_task from celery.apps.worker import logger from django.db.models import Sum, Q -from django.forms import model_to_dict from django.core.cache import cache from dongtai_common.engine.vul_engine import VulEngine from dongtai_common.models import User @@ -23,10 +21,8 @@ from dongtai_common.models.asset import Asset from dongtai_common.models.errorlog import IastErrorlog from dongtai_common.models.heartbeat import IastHeartbeat -from dongtai_common.models.hook_type import HookType from dongtai_common.models.replay_method_pool import IastAgentMethodPoolReplay from dongtai_common.models.replay_queue import IastReplayQueue -from dongtai_common.models.sca_maven_db import ScaMavenDb from dongtai_common.models.vul_level import IastVulLevel from dongtai_common.models.vulnerablity import IastVulnerabilityModel from dongtai_common.utils import const @@ -38,12 +34,8 @@ from dongtai_engine.replay import Replay from dongtai_conf import settings from dongtai_web.dongtai_sca.utils import sca_scan_asset -from dongtai_engine.signals import vul_found -from dongtai_common.models.project_report import ProjectReport import requests -from hashlib import sha1 from dongtai_engine.task_base import replay_payload_data -from typing import List, Dict from dongtai_engine.common.queryset import get_scan_id, load_sink_strategy, get_agent from dongtai_engine.plugins.project_time_update import project_time_stamp_update @@ -577,7 +569,7 @@ def vul_recheck(): header_raw[index] = f'{_header_name}:{recheck_payload}' break try: - headers = base64.b64encode('\n'.join(header_raw).encode("utf-8")) + headers = base64.b64encode('\n'.join(header_raw).encode("raw_unicode_escape")) except Exception as e: logger.warning(f'请求头解析失败，漏洞ID: {vulnerability["id"]}', exc_info=e) elif position == 'COOKIE': @@ -604,7 +596,7 @@ def vul_recheck(): cookie_raw = ';'.join(cookie_raw_items) header_raw[cookie_index] = cookie_raw try: - headers = base64.b64encode('\n'.join(header_raw).encode("utf-8")) + headers = base64.b64encode('\n'.join(header_raw).encode("raw_unicode_escape")) except Exception as e: logger.error(f'请求头解析失败，漏洞ID: {vulnerability["id"]}') diff --git a/dongtai_web/dongtai_sca/models.py b/dongtai_web/dongtai_sca/models.py index 6ea73ee1b..444c62012 100644 --- a/dongtai_web/dongtai_sca/models.py +++ b/dongtai_web/dongtai_sca/models.py @@ -13,7 +13,7 @@ class Package(models.Model): name = models.CharField(max_length=255, blank=True, null=True) version = models.CharField(max_length=255, blank=True, null=True) license = models.CharField(max_length=50, blank=True, null=True) - language = models.CharField(max_length=50, null=False, default='') + language = models.CharField(max_length=50, blank=True, null=False) version_publish_time = models.DateTimeField(blank=True, null=True) created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True) @@ -106,7 +106,7 @@ class VulCveRelation(models.Model): cpe_list = models.JSONField(blank=True, null=True) cvss2_list = models.JSONField(blank=True, null=True) cvss3_list = models.JSONField(blank=True, null=True) - severity = models.CharField(max_length=32, null=False, default='') + severity = models.CharField(max_length=32, blank=True, null=False) publish_time = models.DateTimeField(blank=True, null=True) update_time = models.DateTimeField(blank=True, null=True) created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True) @@ -117,19 +117,19 @@ class Meta: class PackageRepoDependency(models.Model): - repo_aql = models.CharField(max_length=255, null=False, default='') - dependency_aql = models.CharField(max_length=255, null=False, default='') + repo_aql = models.CharField(max_length=255, blank=True, null=False) + dependency_aql = models.CharField(max_length=255, blank=True, null=False) class Meta: db_table = 'sca2_package_repo_dependency' class PackageDependency(models.Model): - package_name = models.CharField(max_length=255, null=False, default='') - p_version = models.CharField(max_length=64, null=False, default='') - dependency_package_name = models.CharField(max_length=255, null=False, default='') - d_version = models.CharField(max_length=64, null=False, default='') - ecosystem = models.CharField(max_length=64, null=False, default='') + package_name = models.CharField(max_length=255, blank=True, null=False) + p_version = models.CharField(max_length=64, blank=True, null=False) + dependency_package_name = models.CharField(max_length=255, blank=True, null=False) + d_version = models.CharField(max_length=64, blank=True, null=False) + ecosystem = models.CharField(max_length=64, blank=True, null=False) class Meta: db_table = 'sca2_package_dependency' diff --git a/dongtai_web/threshold/config_setting.py b/dongtai_web/threshold/config_setting.py index 81aa86344..49525743e 100644 --- a/dongtai_web/threshold/config_setting.py +++ b/dongtai_web/threshold/config_setting.py @@ -188,7 +188,8 @@ def config_update(data, config_id): IastCircuitMetric.objects.filter( circuit_config_id=config_id).delete() obj = IastCircuitConfig.objects.filter(pk=config_id).first() - assert obj is not None + if obj is None: + return for i in data['targets']: create_target(i, obj) @@ -331,8 +332,8 @@ def update(self, request, pk): def reset(self, request, pk): if IastCircuitConfig.objects.filter(pk=pk).exists(): config = IastCircuitConfig.objects.filter(pk=pk, ).first() - assert config is not None - assert config.metric_group is not None + if config is None: + return R.failure() mg = MetricGroup(config.metric_group) data = DEFAULT_CIRCUITCONFIG[mg.name] config_update(data, pk) From e9169577f4825577476700cf05c1ebf14996f6fa Mon Sep 17 00:00:00 2001 From: st1020 Date: Thu, 29 Jun 2023 10:55:01 +0800 Subject: [PATCH 022/161] feat: add project status update task --- dongtai_common/models/project.py | 9 +++ dongtai_conf/celery.py | 10 +--- dongtai_engine/plugins/project_status.py | 72 ++++++++++++++++++++++++ dongtai_web/serializers/project.py | 10 ++-- 4 files changed, 88 insertions(+), 13 deletions(-) create mode 100644 dongtai_engine/plugins/project_status.py diff --git a/dongtai_common/models/project.py b/dongtai_common/models/project.py index 9ce04a185..79779960e 100644 --- a/dongtai_common/models/project.py +++ b/dongtai_common/models/project.py @@ -20,6 +20,12 @@ class VulValidation(models.IntegerChoices): __empty__ = 0 +class ProjectStatus(models.IntegerChoices): + NORMAL = 0 + ERROR = 1 + OFFLINE = 2 + __empty__ = 0 + class IastProjectTemplate(models.Model): template_name = models.CharField(max_length=255) latest_time = models.IntegerField(default=lambda: int(time.time())) @@ -48,6 +54,7 @@ def to_full_project_args(self): } class IastProject(models.Model): + id = models.BigAutoField(primary_key=True) name = models.CharField(max_length=255, blank=True) mode = models.CharField(default="插桩模式", max_length=255, blank=True) vul_count = models.PositiveIntegerField(blank=True, null=True) @@ -71,6 +78,8 @@ class IastProject(models.Model): template = models.ForeignKey(IastProjectTemplate, models.DO_NOTHING) enable_log = models.BooleanField(null=True) log_level = models.CharField(max_length=511, null=True, blank=True) + last_has_online_agent_time = models.IntegerField(default=-1) + status = models.IntegerField(default=0, choices=ProjectStatus.choices) class Meta: managed = get_managed() diff --git a/dongtai_conf/celery.py b/dongtai_conf/celery.py index 1c82272b2..dbca3ecc9 100644 --- a/dongtai_conf/celery.py +++ b/dongtai_conf/celery.py @@ -84,6 +84,7 @@ "dongtai_engine.tasks.vul_recheck": {'exchange': 'dongtai-periodic-task', 'routing_key': 'dongtai-periodic-task'}, "dongtai_engine.preheat.function_preheat": {'exchange': 'dongtai-periodic-task', 'routing_key': 'dongtai-periodic-task'}, "dongtai_engine.plugins.data_clean": {'exchange': 'dongtai-periodic-task', 'routing_key': 'dongtai-periodic-task'}, + "dongtai_engine.plugins.project_status": {'exchange': 'dongtai-periodic-task', 'routing_key': 'dongtai-periodic-task'}, } configs["CELERY_ENABLE_UTC"] = False configs["timezone"] = settings.TIME_ZONE @@ -106,14 +107,9 @@ def ready(self): app.ready = ready print(f"preheat settings now : {DONGTAI_CELERY_CACHE_PREHEAT}") def checkout_preheat_online(status): - from django_celery_beat.models import ( - CrontabSchedule, - PeriodicTask, - IntervalSchedule, - - ) + from django_celery_beat.models import PeriodicTask, IntervalSchedule import json - from datetime import datetime, timedelta + if not status: PeriodicTask.objects.delete(name='preheat functions') else: diff --git a/dongtai_engine/plugins/project_status.py b/dongtai_engine/plugins/project_status.py new file mode 100644 index 000000000..ea58e06c7 --- /dev/null +++ b/dongtai_engine/plugins/project_status.py @@ -0,0 +1,72 @@ +import json +from time import time + +from celery import shared_task +from celery.apps.worker import logger + +from dongtai_common.models.agent import IastAgent +from dongtai_common.models.profile import IastProfile +from dongtai_common.models.project import IastProject, ProjectStatus + + +PROJECT_WARNING_TIME_KEY = "project_warning_time" +DEFAULT_PROJECT_WARNING_TIME = {"error_time": 2 * 7, "offline_time": 3 * 30} + + +def get_project_warning_time() -> dict[str, int]: + profile = ( + IastProfile.objects.filter(key=PROJECT_WARNING_TIME_KEY) + .values_list("value", flat=True) + .first() + ) + if profile is None: + IastProfile( + key=PROJECT_WARNING_TIME_KEY, + value=json.dumps(DEFAULT_PROJECT_WARNING_TIME), + ).save() + return DEFAULT_PROJECT_WARNING_TIME + return json.loads(profile) + + +@shared_task(queue="dongtai-periodic-task") +def update_project_status() -> None: + logger.info("检测项目状态更新开始") + for project in IastProject.objects.all(): + online_agent_count = IastAgent.objects.filter( + bind_project=project, online=True + ).count() + + old_status = project.status + + if online_agent_count == 0: + if project.last_has_online_agent_time == -1: + project.last_has_online_agent_time = int(time()) + project.save(update_fields=("last_has_online_agent_time")) + continue + + dt = int(time()) - project.last_has_online_agent_time + project_warning_time = get_project_warning_time() + error_time = project_warning_time["error_time"] + offline_time = project_warning_time["offline_time"] + + if dt < error_time: + project.status = ProjectStatus.NORMAL + project.save(update_fields=("status",)) + elif dt < offline_time: + project.status = ProjectStatus.ERROR + project.save(update_fields=("status",)) + else: + project.status = ProjectStatus.OFFLINE + project.save(update_fields=("status",)) + else: + project.last_has_online_agent_time = int(time()) + project.status = ProjectStatus.NORMAL + project.save(update_fields=("status", "last_has_online_agent_time")) + + if old_status != project.status: + logger.info( + "update project status: " + f"{project} from {old_status} to {project.status}" + ) + + logger.info("检测项目状态更新结束") diff --git a/dongtai_web/serializers/project.py b/dongtai_web/serializers/project.py index 5eb0ed927..88ee7a67e 100644 --- a/dongtai_web/serializers/project.py +++ b/dongtai_web/serializers/project.py @@ -7,15 +7,13 @@ from rest_framework import serializers from dongtai_common.models.agent import IastAgent -from dongtai_common.models.project import (IastProject, VulValidation) -from dongtai_common.models.vul_level import IastVulLevel +from dongtai_common.models.project import IastProject from dongtai_common.models.vulnerablity import IastVulnerabilityModel -from dongtai_common.models.vulnerablity import IastVulnerabilityStatus -from dongtai_common.utils import const -from dongtai_common.utils.systemsettings import get_vul_validate + from django.db.models import QuerySet from collections import defaultdict from typing import TYPE_CHECKING + if TYPE_CHECKING: from django.core.paginator import _SupportsPagination @@ -66,7 +64,7 @@ class Meta: model = IastProject fields = [ 'id', 'name', 'mode', 'vul_count', 'agent_count', 'owner', - 'latest_time', 'agent_language', 'vul_validation' + 'latest_time', 'agent_language', 'vul_validation', 'status' ] def get_agents(self, obj): From 6aeaa1f3ff3a37cc2605a92e5d07e733b5979ad6 Mon Sep 17 00:00:00 2001 From: st1020 Date: Thu, 29 Jun 2023 12:26:32 +0800 Subject: [PATCH 023/161] feat: add project status update task --- dongtai_web/apps.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dongtai_web/apps.py b/dongtai_web/apps.py index a2671bd4e..660f26efd 100644 --- a/dongtai_web/apps.py +++ b/dongtai_web/apps.py @@ -12,6 +12,10 @@ def ready(self): from dongtai_common.utils.validate import validate_hook_strategy_update from deploy.commands.management.commands.load_hook_strategy import Command from dongtai_conf.settings import AUTO_UPDATE_HOOK_STRATEGY + + # do not remove this import, used in celery + from dongtai_engine.plugins.project_status import update_project_status + if AUTO_UPDATE_HOOK_STRATEGY and not validate_hook_strategy_update(): print("enable auto_update_hook_strategy updating hook strategy from file") Command().handle() From 353f221556796b9bf0eb87c683bca2821af8bc98 Mon Sep 17 00:00:00 2001 From: st1020 Date: Thu, 29 Jun 2023 16:51:40 +0800 Subject: [PATCH 024/161] feat: add project stauts filter --- dongtai_web/views/projects.py | 48 ++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/dongtai_web/views/projects.py b/dongtai_web/views/projects.py index 8b72a99ee..a5be77197 100644 --- a/dongtai_web/views/projects.py +++ b/dongtai_web/views/projects.py @@ -6,7 +6,6 @@ import logging -from dongtai_common.models.agent import IastAgent from dongtai_common.endpoint import R from dongtai_common.endpoint import UserEndPoint from dongtai_common.models.project import IastProject @@ -25,13 +24,19 @@ class _ProjectsArgsSerializer(serializers.Serializer): - pageSize = serializers.IntegerField(default=20, - help_text=_('Number per page')) - page = serializers.IntegerField(min_value=1, default=1, help_text=_('Page index')) + pageSize = serializers.IntegerField(default=20, help_text=_("Number per page")) + page = serializers.IntegerField(min_value=1, default=1, help_text=_("Page index")) name = serializers.CharField( default=None, - help_text=_( - "The name of the item to be searched, supports fuzzy search.")) + help_text=_("The name of the item to be searched, supports fuzzy search."), + ) + status = serializers.IntegerField( + default=None, + allow_null=True, + min_value=0, + max_value=10, + help_text=_("The project status."), + ) _SuccessSerializer = get_response_serializer(ProjectSerializer(many=True)) @@ -43,28 +48,35 @@ class Projects(UserEndPoint): @extend_schema_with_envcheck( [_ProjectsArgsSerializer], - tags=[_('Project')], - summary=_('Projects List'), - description=_("Get the item corresponding to the user, support fuzzy search based on name." - ), + tags=[_("Project")], + summary=_("Projects List"), + description=_( + "Get the item corresponding to the user, support fuzzy search based on name." + ), response_schema=_SuccessSerializer, ) def get(self, request): ser = _ProjectsArgsSerializer(data=request.GET) try: if ser.is_valid(True): - page = ser.validated_data.get('page', 1) - page_size = ser.validated_data.get('pageSize', 20) - name = ser.validated_data.get('name') + page: int = ser.validated_data.get("page", 1) + page_size: int = ser.validated_data.get("pageSize", 20) + name: str = ser.validated_data.get("name") + status: int | None = ser.validated_data.get("status") + else: + return R.failure(data="Can not validation data.") except ValidationError as e: return R.failure(data=e.detail) # users = self.get_auth_users(request.user) department = request.user.get_relative_department() - queryset = IastProject.objects.filter( - department__in=department).order_by('-latest_time') + queryset = IastProject.objects.filter(department__in=department).order_by( + "-latest_time" + ) if name: queryset = queryset.filter(name__icontains=name) + if status is not None: + queryset = queryset.filter(status=status) page_summary, page_data = self.get_paginator(queryset, page, page_size) vul_levels_dict = get_vul_levels_dict(page_data) project_language_dict = get_project_language(page_data) @@ -74,9 +86,9 @@ def get(self, request): page_data, many=True, context={ - 'vul_levels_dict': vul_levels_dict, - 'project_language_dict': project_language_dict, - 'agent_count_dict': agent_count_dict, + "vul_levels_dict": vul_levels_dict, + "project_language_dict": project_language_dict, + "agent_count_dict": agent_count_dict, }, ).data, page=page_summary, From b7a3e3340b4005616c7ac4646beffd2e27739529 Mon Sep 17 00:00:00 2001 From: st1020 Date: Thu, 29 Jun 2023 17:46:46 +0800 Subject: [PATCH 025/161] faet: add project warning time config --- dongtai_web/systemmonitor/project_warning.py | 54 ++++++++++++++++++++ dongtai_web/systemmonitor/urls.py | 13 +++-- 2 files changed, 63 insertions(+), 4 deletions(-) create mode 100644 dongtai_web/systemmonitor/project_warning.py diff --git a/dongtai_web/systemmonitor/project_warning.py b/dongtai_web/systemmonitor/project_warning.py new file mode 100644 index 000000000..69f0381d9 --- /dev/null +++ b/dongtai_web/systemmonitor/project_warning.py @@ -0,0 +1,54 @@ +import json + +from django.http import JsonResponse +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers +from rest_framework.request import Request + +from dongtai_common.endpoint import R, UserEndPoint +from dongtai_common.models.profile import IastProfile +from dongtai_engine.plugins.project_status import ( + PROJECT_WARNING_TIME_KEY, + get_project_warning_time, +) +from dongtai_web.utils import extend_schema_with_envcheck + + +class ProjectWarningSettingsSer(serializers.Serializer): + error_time = serializers.IntegerField() + offline_time = serializers.IntegerField() + + +class ProjectWarningEndpoint(UserEndPoint): + @extend_schema_with_envcheck( + summary=_("Get Profile"), + description=_("Get Profile with key"), + tags=[_("Profile")], + ) + def get(self, request: Request) -> JsonResponse: + return R.success(data=get_project_warning_time()) + + @extend_schema_with_envcheck( + summary=_("Profile modify"), + request=ProjectWarningSettingsSer, + description=_("Modifiy Profile with key"), + tags=[_("Profile")], + ) + def post(self, request: Request) -> JsonResponse: + ser = ProjectWarningSettingsSer(data=request.data) + try: + if ser.is_valid(True): + pass + except serializers.ValidationError as e: + return R.failure(data=e.detail) + + try: + IastProfile.objects.update_or_create( + {"key": PROJECT_WARNING_TIME_KEY, "value": json.dumps(ser.data)}, + key=PROJECT_WARNING_TIME_KEY, + ) + except Exception as e: + print(e) + return R.failure(msg=_("Update {} failed").format(PROJECT_WARNING_TIME_KEY)) + + return R.success(data=ser.data) diff --git a/dongtai_web/systemmonitor/urls.py b/dongtai_web/systemmonitor/urls.py index 4896e07be..756e1c4dd 100644 --- a/dongtai_web/systemmonitor/urls.py +++ b/dongtai_web/systemmonitor/urls.py @@ -1,15 +1,20 @@ from django.urls import include, path from rest_framework import routers -from dongtai_web.systemmonitor.data_clean import DataCleanEndpoint, DataCleanDoItNowEndpoint +from dongtai_web.systemmonitor.data_clean import ( + DataCleanEndpoint, + DataCleanDoItNowEndpoint, +) +from dongtai_web.systemmonitor.project_warning import ProjectWarningEndpoint router = routers.DefaultRouter() base_urlpatterns = [ - path('data_clean', DataCleanEndpoint.as_view()), - path('data_clean/task', DataCleanDoItNowEndpoint.as_view()), + path("data_clean", DataCleanEndpoint.as_view()), + path("data_clean/task", DataCleanDoItNowEndpoint.as_view()), + path("project_warning", ProjectWarningEndpoint.as_view()), ] urlpatterns = [ - path('api/v1/systemmonitor/', include(base_urlpatterns)), + path("api/v1/systemmonitor/", include(base_urlpatterns)), ] From 7409bf666211d0a71525f13687ce0432897a7e53 Mon Sep 17 00:00:00 2001 From: Tscuite <64051240+tscuite@users.noreply.github.com> Date: Thu, 29 Jun 2023 18:14:25 +0800 Subject: [PATCH 026/161] Update dongtai-pr.yaml --- .github/workflows/dongtai-pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dongtai-pr.yaml b/.github/workflows/dongtai-pr.yaml index 584375c3c..abb665249 100644 --- a/.github/workflows/dongtai-pr.yaml +++ b/.github/workflows/dongtai-pr.yaml @@ -17,7 +17,7 @@ jobs: git push --set-upstream origin pr@develop@v${{github.run_number}} generic_handler: - if: ${{ github.repository_owner == 'HXSecurityBusiness' }} + if: ${{ github.repository_owner == 'HXSecurityBusiness' && github.event_name == 'pull_request' }} name: pr runs-on: ubuntu-latest steps: From 9c8290be191a28cc1818b06ab458050ede05fb0f Mon Sep 17 00:00:00 2001 From: Tscuite <64051240+tscuite@users.noreply.github.com> Date: Thu, 29 Jun 2023 18:36:21 +0800 Subject: [PATCH 027/161] Update dongtai-pr.yaml --- .github/workflows/dongtai-pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dongtai-pr.yaml b/.github/workflows/dongtai-pr.yaml index abb665249..584375c3c 100644 --- a/.github/workflows/dongtai-pr.yaml +++ b/.github/workflows/dongtai-pr.yaml @@ -17,7 +17,7 @@ jobs: git push --set-upstream origin pr@develop@v${{github.run_number}} generic_handler: - if: ${{ github.repository_owner == 'HXSecurityBusiness' && github.event_name == 'pull_request' }} + if: ${{ github.repository_owner == 'HXSecurityBusiness' }} name: pr runs-on: ubuntu-latest steps: From 22727f6e96454e01d7156e5ee0e5bc63b77e5e24 Mon Sep 17 00:00:00 2001 From: st1020 Date: Thu, 29 Jun 2023 19:05:47 +0800 Subject: [PATCH 028/161] feat: add project exclude vul status filter --- dongtai_web/serializers/project.py | 34 +++++++++++++++++++++--------- dongtai_web/views/projects.py | 15 ++++++++++++- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/dongtai_web/serializers/project.py b/dongtai_web/serializers/project.py index 88ee7a67e..67cc9ed1f 100644 --- a/dongtai_web/serializers/project.py +++ b/dongtai_web/serializers/project.py @@ -3,7 +3,7 @@ # author:owefsad # software: PyCharm # project: lingzhi-webapi -from django.db.models import Count +from django.db.models import Count, Q from rest_framework import serializers from dongtai_common.models.agent import IastAgent @@ -18,12 +18,16 @@ from django.core.paginator import _SupportsPagination -def get_vul_levels_dict(queryset: "QuerySet | _SupportsPagination") -> defaultdict: +def get_vul_levels_dict( + queryset: "QuerySet | _SupportsPagination", exclude_vul_status: int | None +) -> defaultdict: vul_levels = IastVulnerabilityModel.objects.values( - 'level__name_value', 'level', 'project_id').filter( - project_id__in=list( - queryset.values_list('id', flat=True)), - is_del=0).annotate(total=Count('level')) + "level__name_value", "level", "project_id" + ).filter( + ~Q(status_id=exclude_vul_status), + project_id__in=list(queryset.values_list("id", flat=True)), + is_del=0, + ).annotate(total=Count("level")) vul_levels_dict = defaultdict(list) for k in vul_levels: k['agent__bind_project_id'] = k['project_id'] @@ -51,6 +55,13 @@ def get_agent_count(queryset: "QuerySet | _SupportsPagination") -> defaultdict: return agent_count_dict class ProjectSerializer(serializers.ModelSerializer): + def __init__(self, *args, **kwargs): + if "exclude_vul_status" in kwargs: + self.exclude_vul_status = kwargs.pop("exclude_vul_status") + else: + self.exclude_vul_status = None + super().__init__(*args, **kwargs) + vul_count = serializers.SerializerMethodField( help_text="Vulnerability Count") agent_count = serializers.SerializerMethodField( @@ -70,7 +81,7 @@ class Meta: def get_agents(self, obj): try: all_agents = getattr(obj, 'project_agents') - except Exception as agent_not_found: + except Exception: all_agents = obj.iastagent_set.all() setattr(obj, 'project_agents', all_agents) return all_agents @@ -80,9 +91,12 @@ def get_vul_count(self, obj) -> list: vul_levels = self.context['vul_levels_dict'][obj.id] else: vul_levels = IastVulnerabilityModel.objects.values( - 'level__name_value', - 'level').filter(project_id=obj.id, - is_del=0).annotate(total=Count('level')) + "level__name_value", "level" + ).filter( + ~Q(status_id=self.exclude_vul_status), + project_id=obj.id, + is_del=0, + ).annotate(total=Count("level")) for vul_level in vul_levels: vul_level['name'] = vul_level['level__name_value'] return list(vul_levels) if vul_levels else list() diff --git a/dongtai_web/views/projects.py b/dongtai_web/views/projects.py index a5be77197..b734d2923 100644 --- a/dongtai_web/views/projects.py +++ b/dongtai_web/views/projects.py @@ -37,6 +37,13 @@ class _ProjectsArgsSerializer(serializers.Serializer): max_value=10, help_text=_("The project status."), ) + exclude_vul_status = serializers.IntegerField( + default=None, + allow_null=True, + min_value=0, + max_value=10, + help_text=_("The exclude vulnerability status."), + ) _SuccessSerializer = get_response_serializer(ProjectSerializer(many=True)) @@ -63,6 +70,9 @@ def get(self, request): page_size: int = ser.validated_data.get("pageSize", 20) name: str = ser.validated_data.get("name") status: int | None = ser.validated_data.get("status") + exclude_vul_status: int | None = ser.validated_data.get( + "exclude_vul_status" + ) else: return R.failure(data="Can not validation data.") except ValidationError as e: @@ -78,7 +88,9 @@ def get(self, request): if status is not None: queryset = queryset.filter(status=status) page_summary, page_data = self.get_paginator(queryset, page, page_size) - vul_levels_dict = get_vul_levels_dict(page_data) + vul_levels_dict = get_vul_levels_dict( + page_data, exclude_vul_status=exclude_vul_status + ) project_language_dict = get_project_language(page_data) agent_count_dict = get_agent_count(page_data) return R.success( @@ -90,6 +102,7 @@ def get(self, request): "project_language_dict": project_language_dict, "agent_count_dict": agent_count_dict, }, + exclude_vul_status=exclude_vul_status, ).data, page=page_summary, ) From 13c09d8ab9f794c14cf8b49275c6c297b1a600c7 Mon Sep 17 00:00:00 2001 From: st1020 Date: Thu, 29 Jun 2023 19:19:05 +0800 Subject: [PATCH 029/161] feat: add project exclude vul status filter --- dongtai_web/serializers/project.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dongtai_web/serializers/project.py b/dongtai_web/serializers/project.py index 67cc9ed1f..82ed8fa78 100644 --- a/dongtai_web/serializers/project.py +++ b/dongtai_web/serializers/project.py @@ -24,7 +24,7 @@ def get_vul_levels_dict( vul_levels = IastVulnerabilityModel.objects.values( "level__name_value", "level", "project_id" ).filter( - ~Q(status_id=exclude_vul_status), + ~Q(status_id=exclude_vul_status) if exclude_vul_status is not None else Q(), project_id__in=list(queryset.values_list("id", flat=True)), is_del=0, ).annotate(total=Count("level")) @@ -93,7 +93,8 @@ def get_vul_count(self, obj) -> list: vul_levels = IastVulnerabilityModel.objects.values( "level__name_value", "level" ).filter( - ~Q(status_id=self.exclude_vul_status), + ~Q(status_id=self.exclude_vul_status) + if self.exclude_vul_status is not None else Q(), project_id=obj.id, is_del=0, ).annotate(total=Count("level")) From 7960af308d8d9348c0efabf38ffae2eee65b110a Mon Sep 17 00:00:00 2001 From: st1020 Date: Fri, 30 Jun 2023 11:07:27 +0800 Subject: [PATCH 030/161] feat: remove some vul status --- dongtai_common/utils/const.py | 2 -- dongtai_conf/settings.py | 2 -- .../signals/handlers/vul_handler.py | 2 +- dongtai_engine/tasks.py | 24 ++++++++----------- dongtai_web/views/vul_recheck_v2.py | 4 +--- 5 files changed, 12 insertions(+), 22 deletions(-) diff --git a/dongtai_common/utils/const.py b/dongtai_common/utils/const.py index d4ac70f27..87b486e27 100644 --- a/dongtai_common/utils/const.py +++ b/dongtai_common/utils/const.py @@ -84,7 +84,5 @@ MAX_PAGE_SIZE = 50 VUL_PENDING = 1 -VUL_VERIFYING = 2 VUL_CONFIRMED = 3 -VUL_IGNORE = 4 VUL_SOLVED = 5 diff --git a/dongtai_conf/settings.py b/dongtai_conf/settings.py index 462408992..7665350c5 100644 --- a/dongtai_conf/settings.py +++ b/dongtai_conf/settings.py @@ -552,9 +552,7 @@ def safe_execute(default, exception, function, *args): VERSION = 'latest' # CONST PENDING = 1 -VERIFYING = 2 CONFIRMED = 3 -IGNORE = 4 SOLVED = 5 ENGINE_URL = config.get("engine", "url") HEALTH_ENGINE_URL = urljoin(ENGINE_URL, "/api/engine/health") diff --git a/dongtai_engine/signals/handlers/vul_handler.py b/dongtai_engine/signals/handlers/vul_handler.py index 744261eba..f90f38476 100644 --- a/dongtai_engine/signals/handlers/vul_handler.py +++ b/dongtai_engine/signals/handlers/vul_handler.py @@ -430,7 +430,7 @@ def handler_replay_vul(vul_meta, vul_level, strategy_id, vul_stack, top_stack, bottom_stack, **kwargs): timestamp = int(time.time()) vul = IastVulnerabilityModel.objects.filter( - Q(pk=kwargs['relation_id']) & ~Q(status_id=(3, 4, 5, 6))).first() + Q(pk=kwargs['relation_id']) & ~Q(status_id=(3, 5, 6))).first() logger.info( f'handle vul replay, current strategy:{vul.strategy_id}, target hook_type:{strategy_id}' ) diff --git a/dongtai_engine/tasks.py b/dongtai_engine/tasks.py index 5b4d6f3a6..c1a93f565 100644 --- a/dongtai_engine/tasks.py +++ b/dongtai_engine/tasks.py @@ -13,7 +13,7 @@ from celery import shared_task from celery.apps.worker import logger -from django.db.models import Sum, Q +from django.db.models import Sum from django.core.cache import cache from dongtai_common.engine.vul_engine import VulEngine from dongtai_common.models import User @@ -38,6 +38,7 @@ from dongtai_engine.task_base import replay_payload_data from dongtai_engine.common.queryset import get_scan_id, load_sink_strategy, get_agent from dongtai_engine.plugins.project_time_update import project_time_stamp_update +from dongtai_web.vul_log.vul_log import log_recheck_vul RETRY_INTERVALS = [10, 30, 90] @@ -145,9 +146,12 @@ def search_and_save_vul(engine: Optional[VulEngine], replay_id = method_pool_model.replay_id relation_id = method_pool_model.relation_id timestamp = int(time.time()) - IastVulnerabilityModel.objects.filter(id=relation_id).update( - status_id=settings.IGNORE, - latest_time=timestamp + vul = IastVulnerabilityModel.objects.filter(id=relation_id).get() + log_recheck_vul( + vul.agent.user.id, + vul.agent.user.username, + [vul.id], + '已忽略', ) IastReplayQueue.objects.filter(id=replay_id).update( state=const.SOLVED, @@ -369,8 +373,7 @@ def update_agent_status(): """ from dongtai_engine.plugins.engine_status_change import after_agent_status_update, before_agent_status_update before_agent_status_update() - logger.info(f'检测引擎状态更新开始') - timestamp = int(time.time()) + logger.info('检测引擎状态更新开始') running_agents_ids = list( IastAgent.objects.values("id").filter(online=1).values_list( 'pk', flat=True).all()) @@ -381,15 +384,8 @@ def update_agent_status(): map(lambda x: int(x.replace("heartbeat-", "")), keys_missing)) IastAgent.objects.filter(id__in=stop_agent_ids).update( is_running=0, is_core_running=0, online=0) - vul_id_qs = IastReplayQueue.objects.filter( - update_time__lte=timestamp - 60 * 5, - verify_time__isnull=True, - replay_type=1).values('relation_id').distinct() - IastVulnerabilityModel.objects.filter( - Q(pk__in=vul_id_qs) - & ~Q(status_id__in=(3, 4, 5, 6))).update(status_id=7) logger.info("update offline agent: %s", stop_agent_ids) - logger.info(f'检测引擎状态更新成功') + logger.info('检测引擎状态更新成功') after_agent_status_update() @shared_task(queue='dongtai-periodic-task') diff --git a/dongtai_web/views/vul_recheck_v2.py b/dongtai_web/views/vul_recheck_v2.py index bb65760df..1b455cd76 100644 --- a/dongtai_web/views/vul_recheck_v2.py +++ b/dongtai_web/views/vul_recheck_v2.py @@ -10,10 +10,8 @@ from dongtai_common.models.project import IastProject from dongtai_common.models.replay_queue import IastReplayQueue from dongtai_common.models.vulnerablity import IastVulnerabilityModel -from dongtai_common.utils.validate import Validate from dongtai_web.aggregation.aggregation_common import turnIntListOfStr from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer -from rest_framework import serializers from dongtai_common.endpoint import R from dongtai_common.utils import const @@ -158,7 +156,7 @@ def post(self, request): queryset = IastVulnerabilityModel.objects.filter( Q(is_del=0, project__department__in=department) - & ~Q(status_id__in=(3, 4, 5, 6))) + & ~Q(status_id__in=(3, 5, 6))) ids_list = turnIntListOfStr(vul_ids) vul_queryset = queryset.filter(id__in=ids_list) From 203a7ccfcfce423bf38b0aecd3d3dd586c1953e1 Mon Sep 17 00:00:00 2001 From: st1020 Date: Fri, 30 Jun 2023 11:21:14 +0800 Subject: [PATCH 031/161] feat: remove some vul status --- dongtai_engine/tasks.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/dongtai_engine/tasks.py b/dongtai_engine/tasks.py index c1a93f565..5535dbc20 100644 --- a/dongtai_engine/tasks.py +++ b/dongtai_engine/tasks.py @@ -13,7 +13,7 @@ from celery import shared_task from celery.apps.worker import logger -from django.db.models import Sum +from django.db.models import Sum, Q from django.core.cache import cache from dongtai_common.engine.vul_engine import VulEngine from dongtai_common.models import User @@ -374,6 +374,7 @@ def update_agent_status(): from dongtai_engine.plugins.engine_status_change import after_agent_status_update, before_agent_status_update before_agent_status_update() logger.info('检测引擎状态更新开始') + timestamp = int(time.time()) running_agents_ids = list( IastAgent.objects.values("id").filter(online=1).values_list( 'pk', flat=True).all()) @@ -384,6 +385,20 @@ def update_agent_status(): map(lambda x: int(x.replace("heartbeat-", "")), keys_missing)) IastAgent.objects.filter(id__in=stop_agent_ids).update( is_running=0, is_core_running=0, online=0) + vul_id_qs = IastReplayQueue.objects.filter( + update_time__lte=timestamp - 60 * 5, + verify_time__isnull=True, + replay_type=1).values('relation_id').distinct() + vuls = IastVulnerabilityModel.objects.filter( + Q(pk__in=vul_id_qs) & ~Q(status_id__in=(3, 4, 5, 6)) + ).all() + for vul in vuls: + log_recheck_vul( + vul.agent.user.id, + vul.agent.user.username, + [vul.id], + "验证失败", + ) logger.info("update offline agent: %s", stop_agent_ids) logger.info('检测引擎状态更新成功') after_agent_status_update() From 24f74e70af496e31e470ab59f59dddb48af59466 Mon Sep 17 00:00:00 2001 From: st1020 Date: Fri, 30 Jun 2023 11:43:58 +0800 Subject: [PATCH 032/161] feat: remove some vul status --- dongtai_engine/tasks.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/dongtai_engine/tasks.py b/dongtai_engine/tasks.py index 5535dbc20..f95a3c3bb 100644 --- a/dongtai_engine/tasks.py +++ b/dongtai_engine/tasks.py @@ -10,6 +10,7 @@ import json import time from json import JSONDecodeError +from itertools import groupby from celery import shared_task from celery.apps.worker import logger @@ -390,13 +391,14 @@ def update_agent_status(): verify_time__isnull=True, replay_type=1).values('relation_id').distinct() vuls = IastVulnerabilityModel.objects.filter( - Q(pk__in=vul_id_qs) & ~Q(status_id__in=(3, 4, 5, 6)) + Q(pk__in=vul_id_qs) & ~Q(status_id__in=(3, 5, 6)) ).all() - for vul in vuls: + for _, vul_list in groupby(vuls, lambda x: x.agent_id): + vul_list = list(vul_list) log_recheck_vul( - vul.agent.user.id, - vul.agent.user.username, - [vul.id], + vul_list[0].agent.user.id, + vul_list[0].agent.user.username, + list(map(lambda x: x.id, vul_list)), "验证失败", ) logger.info("update offline agent: %s", stop_agent_ids) From 041e448565da0348ae1f8a71623e5a1bed01154c Mon Sep 17 00:00:00 2001 From: st1020 Date: Fri, 30 Jun 2023 12:10:16 +0800 Subject: [PATCH 033/161] feat: remove some vul status --- dongtai_common/utils/const.py | 2 ++ dongtai_conf/settings.py | 2 ++ dongtai_engine/tasks.py | 4 ++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/dongtai_common/utils/const.py b/dongtai_common/utils/const.py index 87b486e27..d4ac70f27 100644 --- a/dongtai_common/utils/const.py +++ b/dongtai_common/utils/const.py @@ -84,5 +84,7 @@ MAX_PAGE_SIZE = 50 VUL_PENDING = 1 +VUL_VERIFYING = 2 VUL_CONFIRMED = 3 +VUL_IGNORE = 4 VUL_SOLVED = 5 diff --git a/dongtai_conf/settings.py b/dongtai_conf/settings.py index 7665350c5..462408992 100644 --- a/dongtai_conf/settings.py +++ b/dongtai_conf/settings.py @@ -552,7 +552,9 @@ def safe_execute(default, exception, function, *args): VERSION = 'latest' # CONST PENDING = 1 +VERIFYING = 2 CONFIRMED = 3 +IGNORE = 4 SOLVED = 5 ENGINE_URL = config.get("engine", "url") HEALTH_ENGINE_URL = urljoin(ENGINE_URL, "/api/engine/health") diff --git a/dongtai_engine/tasks.py b/dongtai_engine/tasks.py index f95a3c3bb..3f1069fe9 100644 --- a/dongtai_engine/tasks.py +++ b/dongtai_engine/tasks.py @@ -392,8 +392,8 @@ def update_agent_status(): replay_type=1).values('relation_id').distinct() vuls = IastVulnerabilityModel.objects.filter( Q(pk__in=vul_id_qs) & ~Q(status_id__in=(3, 5, 6)) - ).all() - for _, vul_list in groupby(vuls, lambda x: x.agent_id): + ).select_related("agent__user") + for _, vul_list in groupby(vuls, lambda x: x.agent.user_id): vul_list = list(vul_list) log_recheck_vul( vul_list[0].agent.user.id, From e7d8b1943a6dbfc7c13fc7710534dfc6e24928eb Mon Sep 17 00:00:00 2001 From: st1020 Date: Fri, 30 Jun 2023 14:11:14 +0800 Subject: [PATCH 034/161] feat: add MultiUserTestCase --- test/core/tasks.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/test/core/tasks.py b/test/core/tasks.py index d46421a55..41aca4386 100644 --- a/test/core/tasks.py +++ b/test/core/tasks.py @@ -1,7 +1,10 @@ +import json import unittest +from rest_framework.test import APITestCase from test import DongTaiTestCase - +from test.apiserver.test_agent_base import REGISTER_JSON, METHODPOOL_JSON, gzipdata +from dongtai_common.models.user import User class MyTestCase(DongTaiTestCase): def test_something(self): @@ -105,5 +108,33 @@ def parse_path(self): print(project_cookies) +class MultiUserTestCase(APITestCase): + def test_update_agent_status_multi_user(self): + from dongtai_engine.tasks import update_agent_status + user1 = User.objects.filter(pk=1).get() + assert user1 is not None + self.client.force_authenticate(user=user1) + data1 = self.register_agent(name='test1') + + User(id=2, username="test", phone="123456789").save() + user2 = User.objects.filter(pk=2).get() + assert user2 is not None + self.client.force_authenticate(user=user2) + data2 = self.register_agent(name='test2') + + update_agent_status() + + def register_agent(self, **kwargs): + register_data = REGISTER_JSON + register_data.update(kwargs) + data = gzipdata(REGISTER_JSON) + response = self.client.post('http://testserver/api/v1/agent/register', + data=data, + HTTP_CONTENT_ENCODING='gzip', + content_type='application/json', + ) + return json.loads(response.content)['data'] + + if __name__ == '__main__': unittest.main() From 521d55a22fad7c6142675795e5760cff240731e6 Mon Sep 17 00:00:00 2001 From: st1020 Date: Mon, 3 Jul 2023 14:38:59 +0800 Subject: [PATCH 035/161] feat: add check_schema command and change api log --- dongtai_common/endpoint/__init__.py | 44 +++++++++-- dongtai_common/utils/const.py | 7 ++ dongtai_common/utils/init_schema.py | 34 ++++++++ dongtai_conf/settings.py | 39 +++++---- dongtai_web/apps.py | 3 + dongtai_web/utils.py | 79 +++++++------------ .../debug/management/commands/check_schema.py | 25 ++++++ 7 files changed, 152 insertions(+), 79 deletions(-) create mode 100644 dongtai_common/utils/init_schema.py create mode 100644 test/debug/management/commands/check_schema.py diff --git a/dongtai_common/endpoint/__init__.py b/dongtai_common/endpoint/__init__.py index f492186df..34e3507ec 100644 --- a/dongtai_common/endpoint/__init__.py +++ b/dongtai_common/endpoint/__init__.py @@ -33,6 +33,7 @@ from functools import reduce from operator import ior from rest_framework.exceptions import AuthenticationFailed +from dongtai_common.utils.init_schema import VIEW_CLASS_TO_SCHEMA if TYPE_CHECKING: from django.core.paginator import _SupportsPagination @@ -119,14 +120,41 @@ def dispatch(self, request, *args, **kwargs): **kwargs) if self.request.user is not None and self.request.user.is_active and handler.__module__.startswith( 'dongtai_web') and self.description is not None: - self.log_manager.log_action( - user_id=self.request.user.id, - content_type_id=ContentType.objects.get_or_create( - app_label=self.request.content_type)[0].id, - object_id='', - object_repr='', - action_flag=CHANGE, - change_message=f'访问{self.description}接口') + try: + method = self.request.method + if method is None: + raise ValueError("can not get request method") + operate_method = method + schema = VIEW_CLASS_TO_SCHEMA[self.__class__][method] + tags: list[str] = schema["tags"] + summary: str = schema["summary"] + module_name = tags[0] + operate_tag = list(filter(lambda x: x.startswith("operate-"), tags)) + if operate_tag: + operate_method = operate_tag[0].lstrip("operate-") + + match operate_method: + case "GET": + operate_name = "获取" + case "POST": + operate_name = "新增" + case "PUT": + operate_name = "修改" + case "DELETE": + operate_name = "删除" + case _: + raise ValueError("unknown request method") + + self.log_manager.log_action( + user_id=self.request.user.id, + content_type_id=ContentType.objects.get_or_create( + app_label=self.request.content_type)[0].id, + object_id='', + object_repr='', + action_flag=CHANGE, + change_message=f'{operate_name}{module_name}模块{summary}接口') + except Exception as e: + logger.warning(f"get log info failed: {e}") return self.response def handle_exception(self, exc): diff --git a/dongtai_common/utils/const.py b/dongtai_common/utils/const.py index d4ac70f27..f056a14bf 100644 --- a/dongtai_common/utils/const.py +++ b/dongtai_common/utils/const.py @@ -88,3 +88,10 @@ VUL_CONFIRMED = 3 VUL_IGNORE = 4 VUL_SOLVED = 5 + + +# API 操作 tag +OPERATE_GET = "operate-GET" +OPERATE_POST = "operate-POST" +OPERATE_PUT = "operate-PUT" +OPERATE_DELETE = "operate-DELETE" diff --git a/dongtai_common/utils/init_schema.py b/dongtai_common/utils/init_schema.py new file mode 100644 index 000000000..b696aee75 --- /dev/null +++ b/dongtai_common/utils/init_schema.py @@ -0,0 +1,34 @@ +import os +import re +import logging + +from drf_spectacular.generators import SchemaGenerator +from drf_spectacular.drainage import add_trace_message + +logger = logging.getLogger("django") +VIEW_CLASS_TO_SCHEMA = {} + + +def init_schema() -> None: + generator = SchemaGenerator() + generator._initialise_endpoints() + endpoints = generator._get_paths_and_endpoints() + non_trivial_prefix = len(set([view.__class__ for _, _, _, view in endpoints])) > 1 + if non_trivial_prefix: + path_prefix = os.path.commonpath([path for path, _, _, _ in endpoints]) + path_prefix = re.escape(path_prefix) # guard for RE special chars in path + else: + path_prefix = "/" + if not path_prefix.startswith("^"): + path_prefix = "^" + path_prefix # make sure regex only matches from the start + + for path, path_regex, method, view in endpoints: + try: + with add_trace_message(getattr(view, "__class__", view).__name__): + operation = view.schema.get_operation( + path, path_regex, path_prefix, method, generator.registry + ) + VIEW_CLASS_TO_SCHEMA.setdefault(view.__class__, {}) + VIEW_CLASS_TO_SCHEMA[view.__class__][method.upper()] = operation + except Exception: + logger.error(f"unable to get schema: view {view} of path {path}") diff --git a/dongtai_conf/settings.py b/dongtai_conf/settings.py index 462408992..399ab9e49 100644 --- a/dongtai_conf/settings.py +++ b/dongtai_conf/settings.py @@ -491,30 +491,29 @@ def safe_execute(default, exception, function, *args): if os.getenv('environment', None) == 'TEST' or os.getenv('SAVEEYE', None) == 'TRUE': CAPTCHA_NOISE_FUNCTIONS = ('captcha.helpers.noise_null', ) -if os.getenv('environment', 'PROD') in ('TEST', 'DOC') or os.getenv( - 'DOC', None) == 'TRUE': - from django.utils.translation import gettext_lazy as _ - INSTALLED_APPS.append('drf_spectacular') - SPECTACULAR_SETTINGS = { - 'TITLE': - 'DongTai WebApi Doc', - 'VERSION': - "1.1.0", - 'PREPROCESSING_HOOKS': - ['drf_spectacular.hooks.preprocess_exclude_path_format'], - 'URL_FORMAT_OVERRIDE': - None, - 'DESCRIPTION': - _("""Here is the API documentation in dongtai_conf. The corresponding management part API can be found through the relevant tag. + +from django.utils.translation import gettext_lazy as _ +INSTALLED_APPS.append('drf_spectacular') +SPECTACULAR_SETTINGS = { + 'TITLE': + 'DongTai WebApi Doc', + 'VERSION': + "1.1.0", + 'PREPROCESSING_HOOKS': + ['drf_spectacular.hooks.preprocess_exclude_path_format'], + 'URL_FORMAT_OVERRIDE': + None, + 'DESCRIPTION': + _("""Here is the API documentation in dongtai_conf. The corresponding management part API can be found through the relevant tag. There are two authentication methods. You can obtain csrf_token and sessionid through the login process, or access the corresponding API through the user's corresponding Token. The Token method is recommended here, and users can find it in the Agent installation interface such as -H - 'Authorization: Token {token}', here is the token corresponding to the user, the token method also requires a token like this on the request header.""" - ), - } - REST_FRAMEWORK[ - 'DEFAULT_SCHEMA_CLASS'] = 'drf_spectacular.openapi.AutoSchema' +'Authorization: Token {token}', here is the token corresponding to the user, the token method also requires a token like this on the request header.""" + ), +} +REST_FRAMEWORK[ + 'DEFAULT_SCHEMA_CLASS'] = 'drf_spectacular.openapi.AutoSchema' if os.getenv('environment', None) == 'TEST' or os.getenv('CPROFILE', None) == 'TRUE': diff --git a/dongtai_web/apps.py b/dongtai_web/apps.py index 660f26efd..f9a31287b 100644 --- a/dongtai_web/apps.py +++ b/dongtai_web/apps.py @@ -12,6 +12,7 @@ def ready(self): from dongtai_common.utils.validate import validate_hook_strategy_update from deploy.commands.management.commands.load_hook_strategy import Command from dongtai_conf.settings import AUTO_UPDATE_HOOK_STRATEGY + from dongtai_common.utils.init_schema import init_schema # do not remove this import, used in celery from dongtai_engine.plugins.project_status import update_project_status @@ -20,6 +21,8 @@ def ready(self): print("enable auto_update_hook_strategy updating hook strategy from file") Command().handle() + init_schema() + def register_preheat(): from dongtai_engine.preheat import PreHeatRegister diff --git a/dongtai_web/utils.py b/dongtai_web/utils.py index 7c1b981dc..867727d50 100644 --- a/dongtai_web/utils.py +++ b/dongtai_web/utils.py @@ -25,6 +25,7 @@ from dongtai_common.models.vulnerablity import IastVulnerabilityModel from dongtai_conf.settings import OPENAPI from typing import List, Dict, Union, TYPE_CHECKING +from drf_spectacular.utils import extend_schema if TYPE_CHECKING: from django.db.models.query import QuerySet, ValuesQuerySet @@ -83,60 +84,36 @@ def extend_schema_with_envcheck(querys: list = [], response_schema=None, **kwargs): def myextend_schema(func): - import os - if os.getenv( - 'environment', - None) in ( - 'TEST', - 'DOC') or os.getenv( - 'DOC', - None) == 'TRUE': - from drf_spectacular.utils import extend_schema, OpenApiTypes - from drf_spectacular.utils import OpenApiResponse - parameters = list(filter(lambda x: x, map(_filter_query, querys))) - request_examples = list( - filter(lambda x: x, map(_filter_request_body, request_bodys))) - response_examples = list( - filter(lambda x: x, map(_filter_response_body, response_bodys))) - examples = request_examples + response_examples - if kwargs.get('request', None) and request_examples: - kwargs['request'] = {'application/json': OpenApiTypes.OBJECT} - elif isinstance(kwargs.get('request', None), - SerializerMetaclass): - kwargs['request'] = {'application/json': kwargs['request']} - elif kwargs.get('request', None): - kwargs['request'] = {'application/json': kwargs['request']} - deco = extend_schema( - parameters=parameters, - examples=examples if examples else None, - responses={ - 200: OpenApiResponse( - description=_('The http status codes are both 200, please use the status and msg field returned by the response data to troubleshoot'), # type: ignore - response=response_schema)}, - **kwargs) - funcw = deco(func) - funcw.querys = querys - funcw.request_body = request_bodys if request_bodys else [] - return funcw - return func - - return myextend_schema - - -def extend_schema_with_envcheck_v2(*args, **kwargs): - - def myextend_schema(func): - import os - if os.getenv('environment', None) in ('TEST', 'DOC') or os.getenv( - 'DOC', None) == 'TRUE': - from drf_spectacular.utils import extend_schema, OpenApiParameter, OpenApiExample, OpenApiTypes - deco = extend_schema(*args, **kwargs) - funcw = deco(func) - return funcw - return func + from drf_spectacular.utils import OpenApiResponse, extend_schema, OpenApiTypes + parameters = list(filter(lambda x: x, map(_filter_query, querys))) + request_examples = list( + filter(lambda x: x, map(_filter_request_body, request_bodys))) + response_examples = list( + filter(lambda x: x, map(_filter_response_body, response_bodys))) + examples = request_examples + response_examples + if kwargs.get('request', None) and request_examples: + kwargs['request'] = {'application/json': OpenApiTypes.OBJECT} + elif isinstance(kwargs.get('request', None), + SerializerMetaclass): + kwargs['request'] = {'application/json': kwargs['request']} + elif kwargs.get('request', None): + kwargs['request'] = {'application/json': kwargs['request']} + deco = extend_schema( + parameters=parameters, + examples=examples if examples else None, + responses={ + 200: OpenApiResponse( + description=_('The http status codes are both 200, please use the status and msg field returned by the response data to troubleshoot'), # type: ignore + response=response_schema)}, + **kwargs) + funcw = deco(func) + funcw.querys = querys + funcw.request_body = request_bodys if request_bodys else [] + return funcw return myextend_schema +extend_schema_with_envcheck_v2 = extend_schema def get_response_serializer(data_serializer=None, msg_list=None, diff --git a/test/debug/management/commands/check_schema.py b/test/debug/management/commands/check_schema.py new file mode 100644 index 000000000..5ceb22d4b --- /dev/null +++ b/test/debug/management/commands/check_schema.py @@ -0,0 +1,25 @@ +from dongtai_common.utils.init_schema import VIEW_CLASS_TO_SCHEMA + +from django.core.management.base import BaseCommand + + +class Command(BaseCommand): + help = "Check API schema is complete" + + def handle(self, *args, **options): + count = 0 + for view, schema in VIEW_CLASS_TO_SCHEMA.items(): + for method, schema in schema.items(): + if schema is None: + self.stdout.write(self.style.ERROR(f"No schema: {view}")) + continue + for schema_field in ("tags", "summary"): + if not schema.get(schema_field, None): + count += 1 + self.stdout.write( + self.style.ERROR( + f'[{count}]Miss "{schema_field}" schema: {method} {view}' + ) + ) + + self.stdout.write(self.style.SUCCESS("Check API schema done")) From 3904e245256cd9a619af5cccf098467315273321 Mon Sep 17 00:00:00 2001 From: st1020 Date: Mon, 3 Jul 2023 15:40:36 +0800 Subject: [PATCH 036/161] feat: add check_schema command and change api log --- dongtai_common/endpoint/__init__.py | 43 +++++++++---------- dongtai_conf/settings.py | 2 +- dongtai_web/utils.py | 1 + .../debug/management/commands/check_schema.py | 9 ++++ 4 files changed, 32 insertions(+), 23 deletions(-) diff --git a/dongtai_common/endpoint/__init__.py b/dongtai_common/endpoint/__init__.py index 34e3507ec..23be39474 100644 --- a/dongtai_common/endpoint/__init__.py +++ b/dongtai_common/endpoint/__init__.py @@ -6,7 +6,7 @@ import json import logging -from django.contrib.admin.models import LogEntryManager, LogEntry, CHANGE +from django.contrib.admin.models import LogEntryManager, LogEntry, ADDITION, CHANGE, DELETION from django.contrib.contenttypes.models import ContentType from django.core.paginator import Paginator from django.db.models import QuerySet @@ -125,7 +125,7 @@ def dispatch(self, request, *args, **kwargs): if method is None: raise ValueError("can not get request method") operate_method = method - schema = VIEW_CLASS_TO_SCHEMA[self.__class__][method] + schema = VIEW_CLASS_TO_SCHEMA[self.__class__][method] tags: list[str] = schema["tags"] summary: str = schema["summary"] module_name = tags[0] @@ -133,26 +133,25 @@ def dispatch(self, request, *args, **kwargs): if operate_tag: operate_method = operate_tag[0].lstrip("operate-") - match operate_method: - case "GET": - operate_name = "获取" - case "POST": - operate_name = "新增" - case "PUT": - operate_name = "修改" - case "DELETE": - operate_name = "删除" - case _: - raise ValueError("unknown request method") - - self.log_manager.log_action( - user_id=self.request.user.id, - content_type_id=ContentType.objects.get_or_create( - app_label=self.request.content_type)[0].id, - object_id='', - object_repr='', - action_flag=CHANGE, - change_message=f'{operate_name}{module_name}模块{summary}接口') + if operate_method != "GET": + match operate_method: + case "POST": + action_flag = ADDITION + case "PUT": + action_flag = CHANGE + case "DELETE": + action_flag = DELETION + case _: + raise ValueError("unknown request method") + + self.log_manager.log_action( + user_id=self.request.user.id, + content_type_id=ContentType.objects.get_or_create( + app_label=self.request.content_type)[0].id, + object_id='', + object_repr='', + action_flag=action_flag, + change_message=f'{module_name}模块{summary}接口') except Exception as e: logger.warning(f"get log info failed: {e}") return self.response diff --git a/dongtai_conf/settings.py b/dongtai_conf/settings.py index 399ab9e49..2cfc7fe65 100644 --- a/dongtai_conf/settings.py +++ b/dongtai_conf/settings.py @@ -510,7 +510,7 @@ def safe_execute(default, exception, function, *args): The Token method is recommended here, and users can find it in the Agent installation interface such as -H 'Authorization: Token {token}', here is the token corresponding to the user, the token method also requires a token like this on the request header.""" - ), + ), } REST_FRAMEWORK[ 'DEFAULT_SCHEMA_CLASS'] = 'drf_spectacular.openapi.AutoSchema' diff --git a/dongtai_web/utils.py b/dongtai_web/utils.py index 867727d50..f0ab8f696 100644 --- a/dongtai_web/utils.py +++ b/dongtai_web/utils.py @@ -113,6 +113,7 @@ def myextend_schema(func): return myextend_schema + extend_schema_with_envcheck_v2 = extend_schema def get_response_serializer(data_serializer=None, diff --git a/test/debug/management/commands/check_schema.py b/test/debug/management/commands/check_schema.py index 5ceb22d4b..2c8a09644 100644 --- a/test/debug/management/commands/check_schema.py +++ b/test/debug/management/commands/check_schema.py @@ -13,13 +13,22 @@ def handle(self, *args, **options): if schema is None: self.stdout.write(self.style.ERROR(f"No schema: {view}")) continue + has_error = False for schema_field in ("tags", "summary"): if not schema.get(schema_field, None): + has_error = True count += 1 self.stdout.write( self.style.ERROR( f'[{count}]Miss "{schema_field}" schema: {method} {view}' ) ) + if not has_error: + self.stdout.write( + self.style.SUCCESS( + f"{method} {view}: " + f"tags: {schema['tags']}, summary: {schema['summary']}" + ) + ) self.stdout.write(self.style.SUCCESS("Check API schema done")) From 86c028dc098f786c3f5fb79ad0056f293f264be1 Mon Sep 17 00:00:00 2001 From: st1020 Date: Tue, 4 Jul 2023 11:08:36 +0800 Subject: [PATCH 037/161] feat: add check_schema command and change api log --- dongtai_common/endpoint/__init__.py | 52 +++++---- dongtai_common/models/log.py | 31 ++++++ dongtai_common/models/user.py | 1 + dongtai_conf/urls.py | 25 ++--- dongtai_engine/preheat.py | 6 +- dongtai_web/urls.py | 5 + dongtai_web/views/log_export_v2.py | 83 ++++++++++++++ dongtai_web/views/logs_v2.py | 165 ++++++++++++++++++++++++++++ 8 files changed, 330 insertions(+), 38 deletions(-) create mode 100644 dongtai_common/models/log.py create mode 100644 dongtai_web/views/log_export_v2.py create mode 100644 dongtai_web/views/logs_v2.py diff --git a/dongtai_common/endpoint/__init__.py b/dongtai_common/endpoint/__init__.py index 23be39474..d311c57b1 100644 --- a/dongtai_common/endpoint/__init__.py +++ b/dongtai_common/endpoint/__init__.py @@ -6,7 +6,7 @@ import json import logging -from django.contrib.admin.models import LogEntryManager, LogEntry, ADDITION, CHANGE, DELETION +from django.http.request import HttpRequest from django.contrib.contenttypes.models import ContentType from django.core.paginator import Paginator from django.db.models import QuerySet @@ -23,6 +23,7 @@ from dongtai_common.models.asset import Asset from dongtai_common.models.asset_aggr import AssetAggr from dongtai_common.models.asset_vul import IastVulAssetRelation, IastAssetVul +from dongtai_common.models.log import IastLog, OperateType from dongtai_common.permissions import UserPermission, ScopedPermission, SystemAdminPermission, TalentAdminPermission from dongtai_common.utils import const from django.utils.translation import gettext_lazy as _ @@ -57,8 +58,6 @@ def __init__(self, **kwargs): # Go through keyword arguments, and either save their values to our # instance, or raise an error. super().__init__(**kwargs) - self.log_manager = LogEntryManager() - self.log_manager.model = LogEntry def load_json_body(self, request): """ @@ -133,25 +132,25 @@ def dispatch(self, request, *args, **kwargs): if operate_tag: operate_method = operate_tag[0].lstrip("operate-") - if operate_method != "GET": - match operate_method: - case "POST": - action_flag = ADDITION - case "PUT": - action_flag = CHANGE - case "DELETE": - action_flag = DELETION - case _: - raise ValueError("unknown request method") - - self.log_manager.log_action( - user_id=self.request.user.id, - content_type_id=ContentType.objects.get_or_create( - app_label=self.request.content_type)[0].id, - object_id='', - object_repr='', - action_flag=action_flag, - change_message=f'{module_name}模块{summary}接口') + match operate_method: + case "GET": + operate_type = OperateType.GET + case "POST": + operate_type = OperateType.ADD + case "PUT": + operate_type = OperateType.CHANGE + case "DELETE": + operate_type = OperateType.DELETE + case _: + raise ValueError("unknown request method") + + IastLog.objects.create( + module_name=module_name, + function_name=summary, + operate_type=operate_type, + user_id=self.request.user.id, + access_ip=get_client_ip(self.request) + ) except Exception as e: logger.warning(f"get log info failed: {e}") return self.response @@ -430,3 +429,12 @@ def failure(status=202, data=None, status_code=200, msg=_("failure")): resp_data, status=status_code, ) + + +def get_client_ip(request: HttpRequest) -> str | None: + x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR') + if x_forwarded_for: + ip = x_forwarded_for.split(',')[0] + else: + ip = request.META.get('REMOTE_ADDR') + return ip diff --git a/dongtai_common/models/log.py b/dongtai_common/models/log.py new file mode 100644 index 000000000..c678df0b5 --- /dev/null +++ b/dongtai_common/models/log.py @@ -0,0 +1,31 @@ +from django.db import models +from django.utils import timezone +from django.utils.translation import gettext_lazy as _ + +from dongtai_common.models import User +from dongtai_common.utils.settings import get_managed + + +class OperateType(models.IntegerChoices): + GET = 1, _("GET") + ADD = 2, _("ADD") + CHANGE = 3, _("CHANGE") + DELETE = 4, _("DELETE") + + +class IastLog(models.Model): + id = models.BigAutoField(primary_key=True) + action_time = models.DateTimeField( + _("action time"), + default=timezone.now, + editable=False, + ) + module_name = models.CharField(max_length=255) + function_name = models.CharField(max_length=255) + operate_type = models.IntegerField(choices=OperateType.choices) + user = models.ForeignKey(User, on_delete=models.DO_NOTHING) + access_ip = models.CharField(max_length=255, blank=True) + + class Meta: + managed = get_managed() + db_table = "iast_log" diff --git a/dongtai_common/models/user.py b/dongtai_common/models/user.py index 5043ab908..5f0f7b9cd 100644 --- a/dongtai_common/models/user.py +++ b/dongtai_common/models/user.py @@ -69,6 +69,7 @@ def create_system_user(self, class User(AbstractUser, PermissionsMixin): + id = models.BigAutoField(primary_key=True) is_superuser = models.IntegerField(default=0) phone = models.CharField(max_length=15, blank=True) default_language = models.CharField(max_length=15, blank=True) diff --git a/dongtai_conf/urls.py b/dongtai_conf/urls.py index 65aa5a30f..755e7ea77 100644 --- a/dongtai_conf/urls.py +++ b/dongtai_conf/urls.py @@ -25,19 +25,18 @@ ] urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) -if os.getenv('environment', 'PROD') in ('TEST', 'DOC') or os.getenv('DOC', None) == 'TRUE': - from drf_spectacular.views import SpectacularJSONAPIView, SpectacularRedocView, SpectacularSwaggerView - urlpatterns.extend([ - path('api/XZPcGFKoxYXScwGjQtJx8u/schema/', - SpectacularJSONAPIView.as_view(), - name='schema'), - path('api/XZPcGFKoxYXScwGjQtJx8u/schema/swagger-ui/', - SpectacularSwaggerView.as_view(url_name='schema'), - name='swagger-ui'), - path('api/XZPcGFKoxYXScwGjQtJx8u/schema/redoc/', - SpectacularRedocView.as_view(url_name='schema'), - name='redoc'), - ]) +from drf_spectacular.views import SpectacularJSONAPIView, SpectacularRedocView, SpectacularSwaggerView +urlpatterns.extend([ + path('api/XZPcGFKoxYXScwGjQtJx8u/schema/', + SpectacularJSONAPIView.as_view(), + name='schema'), + path('api/XZPcGFKoxYXScwGjQtJx8u/schema/swagger-ui/', + SpectacularSwaggerView.as_view(url_name='schema'), + name='swagger-ui'), + path('api/XZPcGFKoxYXScwGjQtJx8u/schema/redoc/', + SpectacularRedocView.as_view(url_name='schema'), + name='redoc'), +]) if os.getenv('DJANGOSILK', None) == 'TRUE': silk_path = os.getenv( diff --git a/dongtai_engine/preheat.py b/dongtai_engine/preheat.py index 9312a996c..af1c0ae75 100644 --- a/dongtai_engine/preheat.py +++ b/dongtai_engine/preheat.py @@ -19,14 +19,14 @@ @shared_task(queue='dongtai-periodic-task') def function_preheat(): - from django.contrib.admin.models import LogEntry + from dongtai_common.models.log import IastLog time_threshold = datetime.now() - timedelta(hours=1) - need_preheat = LogEntry.objects.filter( + need_preheat = IastLog.objects.filter( action_time__gt=time_threshold).exists() if need_preheat: time_min = datetime.now() - timedelta(hours=72) user_ids = list( - LogEntry.objects.filter(action_time__gt=time_min).values_list( + IastLog.objects.filter(action_time__gt=time_min).values_list( 'user__id', flat=True).distinct().order_by('user__id').all()) logger.info(f"user_ids: {user_ids}") for user_id in user_ids: diff --git a/dongtai_web/urls.py b/dongtai_web/urls.py index ba3be9eda..8f0339f9f 100644 --- a/dongtai_web/urls.py +++ b/dongtai_web/urls.py @@ -67,6 +67,8 @@ from dongtai_web.views.project_version_list import ProjectVersionList from dongtai_web.views.project_version_update import ProjectVersionUpdate from dongtai_web.views.projects import Projects +from dongtai_web.views.logs_v2 import LogsV2Endpoint +from dongtai_web.views.log_export_v2 import LogExportV2 from dongtai_web.views.sca_details import ScaDetailView @@ -472,6 +474,9 @@ path('api/v2/app_vul_summary', GetAppVulsSummary.as_view()), path('api/v2/api_route/search', NewApiRouteSearch.as_view()), path('api/v2/project_version', NewProjectVersionList.as_view()), + # 日志 + path('api/v2/logs', LogsV2Endpoint.as_view()), + path('api/v2/logs/export', LogExportV2.as_view()), ]) # urlpatterns.extend(scaupload_urls) departured diff --git a/dongtai_web/views/log_export_v2.py b/dongtai_web/views/log_export_v2.py new file mode 100644 index 000000000..bef0c1228 --- /dev/null +++ b/dongtai_web/views/log_export_v2.py @@ -0,0 +1,83 @@ +from django.http import HttpResponse +from django.utils.encoding import escape_uri_path +from django.utils.translation import gettext_lazy as _ +from drf_spectacular.utils import extend_schema +from import_export import resources +from rest_framework import serializers +from rest_framework.request import Request + +from dongtai_common.endpoint import R, UserEndPoint +from dongtai_common.models.log import IastLog +from dongtai_common.models.user import User + + +class _LogsDeleteSerializer(serializers.Serializer): + ids = serializers.Field(default=None, help_text=_("Log ids to delete")) + + +class LogResurce(resources.ModelResource): + def get_export_headers(self): + return ["时间", "模块名称", "功能名称", "操作类型", "用户", "访问IP地址"] + + class Meta: + model = IastLog + fields = ( + "action_time", + "module_name", + "function_name", + "operate_type", + "user", + "access_ip", + ) + + +class LogExportV2(UserEndPoint): + resource_class = LogResurce + + @staticmethod + def attachment_response( + export_data, + filename: str = "download.xls", + content_type: str = "application/vnd.ms-excel", + ) -> HttpResponse: + """ + - https://segmentfault.com/q/1010000009719860 + - https://blog.csdn.net/qq_34309753/article/details/99628474 + """ + response = HttpResponse(export_data, content_type=content_type) + response["content_type"] = content_type + response["Content-Disposition"] = "attachment; filename*=utf-8''{}".format( + escape_uri_path(filename) + ) + return response + + @extend_schema( + parameters=[_LogsDeleteSerializer], + description="Export Logs.", + summary="Export Logs", + tags=["Logs"], + ) + def get(self, request: Request) -> HttpResponse: + ids: str | None = request.query_params.get("ids") + if ids: + id_list = [int(id.strip()) for id in ids.split(",")] + user: User = request.user # type: ignore + if user.is_system_admin(): + queryset = IastLog.objects.filter(id__in=id_list).filter() + elif user.is_talent_admin(): + auth_users = UserEndPoint.get_auth_users(user) + queryset = IastLog.objects.filter( + id__in=id_list, user__in=auth_users + ).filter() + else: + return R.failure(msg=_("no permission")) + resources = self.resource_class() + export_data = resources.export(queryset, False) + return self.attachment_response( + getattr(export_data, "xls"), filename="用户操作日志.xls" + ) + else: + return R.failure( + status=202, + msg=_("Export failed, error message: Log id should not be empty"), + ) diff --git a/dongtai_web/views/logs_v2.py b/dongtai_web/views/logs_v2.py new file mode 100644 index 000000000..c79b56595 --- /dev/null +++ b/dongtai_web/views/logs_v2.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python +# -*- coding:utf-8 -*- +# author:sjh +# software: PyCharm +# project: webapi +import logging +from datetime import datetime + +from django.core.cache import cache +from django.http import JsonResponse +from django.utils.translation import gettext_lazy as _ +from drf_spectacular.utils import extend_schema +from rest_framework import serializers +from rest_framework.request import Request +from rest_framework.serializers import ValidationError + +from dongtai_common.endpoint import R, UserEndPoint +from dongtai_common.models.log import IastLog +from dongtai_common.models.user import User + +logger = logging.getLogger("dongtai-webapi") + + +class _LogsArgsSerializer(serializers.Serializer): + pageSize = serializers.IntegerField(default=20, help_text=_("Number per page")) + page = serializers.IntegerField(min_value=1, default=1, help_text=_("Page index")) + startTime = serializers.DateTimeField(default=None, help_text=_("The start time.")) + endTime = serializers.DateTimeField(default=None, help_text=_("The end time.")) + + +class _LogsDeleteSerializer(serializers.Serializer): + ids = serializers.Field(default=None, help_text=_("Log ids to delete")) + + +class LogsV2Endpoint(UserEndPoint): + name = "api-v2-logs" + description = _("Log list") + + def make_key(self, request): + self.cache_key = f"{request.user.id}_total_logs_id" + self.cache_key_max_id = f"{request.user.id}_max_logs_id" + + def get_query_cache(self): + total = cache.get(self.cache_key) + max_id = cache.get(self.cache_key_max_id) + return total, max_id + + def set_query_cache(self, queryset): + total = queryset.values("id").count() + if total > 0: + max_id = queryset.values_list("id", flat=True).order_by("-id")[0] + else: + max_id = 0 + cache.set(self.cache_key, total, 60 * 60) + cache.set(self.cache_key_max_id, max_id, 60 * 60) + return total, max_id + + @extend_schema( + parameters=[_LogsArgsSerializer], + description="Get List of logs.", + summary="Log List", + tags=["Logs"], + ) + def get(self, request: Request) -> JsonResponse: + ser = _LogsArgsSerializer(data=request.GET) + user: User = request.user # type: ignore + try: + if ser.is_valid(True): + page: int = ser.validated_data.get("page", 1) + page_size: int = ser.validated_data.get("pageSize", 20) + start_time: datetime | None = ser.validated_data.get("startTime", None) + end_time: datetime | None = ser.validated_data.get("endTime", None) + else: + return R.failure(data="Can not validation data.") + except ValidationError as e: + return R.failure(data=e.detail) + + try: + if user.is_system_admin(): + queryset = IastLog.objects.all() + elif user.is_talent_admin(): + users = self.get_auth_users(user) + user_ids = list(users.values_list("id", flat=True)) + queryset = IastLog.objects.filter(user_id__in=user_ids) + else: + queryset = IastLog.objects.filter(user=user) + + if start_time is not None and end_time is not None: + queryset = queryset.filter(action_time__range=(start_time, end_time)) + elif start_time is not None: + queryset = queryset.filter(action_time__gte=start_time) + elif end_time is not None: + queryset.filter(action_time__lte=end_time) + + # set cache key + self.make_key(request) + if page == 1: + total, max_id = self.set_query_cache(queryset) + else: + total, max_id = self.get_query_cache() + if not total or not max_id: + total, max_id = self.set_query_cache(queryset) + # only read log_id + cur_data = ( + queryset.filter(id__lte=max_id) + .values_list("id", flat=True) + .order_by("-id")[(page - 1) * page_size : page * page_size] + ) + cur_ids = [] + for item in cur_data: + cur_ids.append(item) + # read log detail + page_data = ( + IastLog.objects.filter(id__in=cur_ids) + .order_by("-id") + .select_related("user") + ) + data = [] + for item in page_data: + data.append( + { + "log_id": item.id, + "user_id": item.user.id, + "username": item.user.username, + "action_time": item.action_time.strftime("%Y-%m-%d %H:%M:%S"), + "module_name": item.module_name, + "function_name": item.function_name, + "operate_type": item.operate_type, + "access_ip": item.access_ip, + } + ) + return R.success(data=data, total=total) + except Exception as e: + logger.error(e, exc_info=True) + return R.success(data=list(), msg=_("failure")) + + @extend_schema( + parameters=[_LogsDeleteSerializer], + description="Delete Logs. " + "Query param ids is optional, if not set ids, will delete all logs", + summary="Delete Logs", + tags=["Logs"], + ) + def delete(self, request: Request) -> JsonResponse: + ids: str | None = request.query_params.get("ids", None) + user: User = request.user # type: ignore + if ids: + id_list = [int(id.strip()) for id in ids.split(",")] + if user.is_superuser == 1: + IastLog.objects.filter(id__in=id_list).delete() + return R.success(msg=_("success")) + users = self.get_auth_users(user) + user_ids = list(users.values_list("id", flat=True)) + IastLog.objects.filter(id__in=id_list, user_id__in=user_ids).delete() + return R.success(msg=_("success")) + else: + now = datetime.now() + if user.is_system_admin(): + IastLog.objects.filter(action_time__lt=now).delete() + elif user.is_talent_admin(): + users = self.get_auth_users(user) + IastLog.objects.filter(action_time__lt=now, user__in=users).delete() + else: + return R.failure(status=203, msg=_("no permission")) + return R.success() From a2b94606e5a6909f28531a7d3274c98636df5951 Mon Sep 17 00:00:00 2001 From: st1020 Date: Tue, 4 Jul 2023 11:43:32 +0800 Subject: [PATCH 038/161] feat: add url field in log model --- dongtai_common/endpoint/__init__.py | 6 ++++- dongtai_common/models/log.py | 2 ++ dongtai_common/utils/init_schema.py | 8 ++++-- dongtai_conf/urls.py | 25 ++++++++++--------- dongtai_web/views/log_export_v2.py | 13 +++++++++- dongtai_web/views/logs_v2.py | 2 ++ .../debug/management/commands/check_schema.py | 1 + 7 files changed, 41 insertions(+), 16 deletions(-) diff --git a/dongtai_common/endpoint/__init__.py b/dongtai_common/endpoint/__init__.py index d311c57b1..66029e11c 100644 --- a/dongtai_common/endpoint/__init__.py +++ b/dongtai_common/endpoint/__init__.py @@ -124,7 +124,9 @@ def dispatch(self, request, *args, **kwargs): if method is None: raise ValueError("can not get request method") operate_method = method - schema = VIEW_CLASS_TO_SCHEMA[self.__class__][method] + path, _path_regex, schema = VIEW_CLASS_TO_SCHEMA[self.__class__][method] + if schema is None: + raise ValueError("can not get schema") tags: list[str] = schema["tags"] summary: str = schema["summary"] module_name = tags[0] @@ -145,6 +147,8 @@ def dispatch(self, request, *args, **kwargs): raise ValueError("unknown request method") IastLog.objects.create( + url=path, + raw_url=self.request.get_full_path(), module_name=module_name, function_name=summary, operate_type=operate_type, diff --git a/dongtai_common/models/log.py b/dongtai_common/models/log.py index c678df0b5..ef838a4b0 100644 --- a/dongtai_common/models/log.py +++ b/dongtai_common/models/log.py @@ -20,6 +20,8 @@ class IastLog(models.Model): default=timezone.now, editable=False, ) + url = models.CharField(max_length=255) + raw_url = models.CharField(max_length=255) module_name = models.CharField(max_length=255) function_name = models.CharField(max_length=255) operate_type = models.IntegerField(choices=OperateType.choices) diff --git a/dongtai_common/utils/init_schema.py b/dongtai_common/utils/init_schema.py index b696aee75..daa736b1a 100644 --- a/dongtai_common/utils/init_schema.py +++ b/dongtai_common/utils/init_schema.py @@ -6,7 +6,7 @@ from drf_spectacular.drainage import add_trace_message logger = logging.getLogger("django") -VIEW_CLASS_TO_SCHEMA = {} +VIEW_CLASS_TO_SCHEMA: dict[type, dict[str, tuple[str, str, dict | None]]] = {} def init_schema() -> None: @@ -29,6 +29,10 @@ def init_schema() -> None: path, path_regex, path_prefix, method, generator.registry ) VIEW_CLASS_TO_SCHEMA.setdefault(view.__class__, {}) - VIEW_CLASS_TO_SCHEMA[view.__class__][method.upper()] = operation + VIEW_CLASS_TO_SCHEMA[view.__class__][method.upper()] = ( + path, + path_regex, + operation, + ) except Exception: logger.error(f"unable to get schema: view {view} of path {path}") diff --git a/dongtai_conf/urls.py b/dongtai_conf/urls.py index 755e7ea77..65aa5a30f 100644 --- a/dongtai_conf/urls.py +++ b/dongtai_conf/urls.py @@ -25,18 +25,19 @@ ] urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) -from drf_spectacular.views import SpectacularJSONAPIView, SpectacularRedocView, SpectacularSwaggerView -urlpatterns.extend([ - path('api/XZPcGFKoxYXScwGjQtJx8u/schema/', - SpectacularJSONAPIView.as_view(), - name='schema'), - path('api/XZPcGFKoxYXScwGjQtJx8u/schema/swagger-ui/', - SpectacularSwaggerView.as_view(url_name='schema'), - name='swagger-ui'), - path('api/XZPcGFKoxYXScwGjQtJx8u/schema/redoc/', - SpectacularRedocView.as_view(url_name='schema'), - name='redoc'), -]) +if os.getenv('environment', 'PROD') in ('TEST', 'DOC') or os.getenv('DOC', None) == 'TRUE': + from drf_spectacular.views import SpectacularJSONAPIView, SpectacularRedocView, SpectacularSwaggerView + urlpatterns.extend([ + path('api/XZPcGFKoxYXScwGjQtJx8u/schema/', + SpectacularJSONAPIView.as_view(), + name='schema'), + path('api/XZPcGFKoxYXScwGjQtJx8u/schema/swagger-ui/', + SpectacularSwaggerView.as_view(url_name='schema'), + name='swagger-ui'), + path('api/XZPcGFKoxYXScwGjQtJx8u/schema/redoc/', + SpectacularRedocView.as_view(url_name='schema'), + name='redoc'), + ]) if os.getenv('DJANGOSILK', None) == 'TRUE': silk_path = os.getenv( diff --git a/dongtai_web/views/log_export_v2.py b/dongtai_web/views/log_export_v2.py index bef0c1228..6c903b4a4 100644 --- a/dongtai_web/views/log_export_v2.py +++ b/dongtai_web/views/log_export_v2.py @@ -17,12 +17,23 @@ class _LogsDeleteSerializer(serializers.Serializer): class LogResurce(resources.ModelResource): def get_export_headers(self): - return ["时间", "模块名称", "功能名称", "操作类型", "用户", "访问IP地址"] + return [ + "时间", + "URL", + "Raw URL", + "模块名称", + "功能名称", + "操作类型", + "用户", + "访问IP地址", + ] class Meta: model = IastLog fields = ( "action_time", + "url", + "raw_url", "module_name", "function_name", "operate_type", diff --git a/dongtai_web/views/logs_v2.py b/dongtai_web/views/logs_v2.py index c79b56595..d68e0fe44 100644 --- a/dongtai_web/views/logs_v2.py +++ b/dongtai_web/views/logs_v2.py @@ -123,6 +123,8 @@ def get(self, request: Request) -> JsonResponse: "user_id": item.user.id, "username": item.user.username, "action_time": item.action_time.strftime("%Y-%m-%d %H:%M:%S"), + "url": item.url, + "raw_url": item.raw_url, "module_name": item.module_name, "function_name": item.function_name, "operate_type": item.operate_type, diff --git a/test/debug/management/commands/check_schema.py b/test/debug/management/commands/check_schema.py index 2c8a09644..b3a53f2d2 100644 --- a/test/debug/management/commands/check_schema.py +++ b/test/debug/management/commands/check_schema.py @@ -10,6 +10,7 @@ def handle(self, *args, **options): count = 0 for view, schema in VIEW_CLASS_TO_SCHEMA.items(): for method, schema in schema.items(): + _, _, schema = schema if schema is None: self.stdout.write(self.style.ERROR(f"No schema: {view}")) continue From fd910ba9aee50433a89da9251ace06c48f1d28da Mon Sep 17 00:00:00 2001 From: st1020 Date: Tue, 4 Jul 2023 12:28:24 +0800 Subject: [PATCH 039/161] feat: add log/api_list and log/ip_list API --- dongtai_web/urls.py | 4 +++ dongtai_web/views/log_api_list.py | 46 +++++++++++++++++++++++++++++++ dongtai_web/views/log_ip_list.py | 46 +++++++++++++++++++++++++++++++ dongtai_web/views/logs_v2.py | 17 ++++++++---- 4 files changed, 107 insertions(+), 6 deletions(-) create mode 100644 dongtai_web/views/log_api_list.py create mode 100644 dongtai_web/views/log_ip_list.py diff --git a/dongtai_web/urls.py b/dongtai_web/urls.py index 8f0339f9f..d016d7376 100644 --- a/dongtai_web/urls.py +++ b/dongtai_web/urls.py @@ -69,6 +69,8 @@ from dongtai_web.views.projects import Projects from dongtai_web.views.logs_v2 import LogsV2Endpoint from dongtai_web.views.log_export_v2 import LogExportV2 +from dongtai_web.views.log_ip_list import LogsIPList +from dongtai_web.views.log_api_list import LogsApiList from dongtai_web.views.sca_details import ScaDetailView @@ -477,6 +479,8 @@ # 日志 path('api/v2/logs', LogsV2Endpoint.as_view()), path('api/v2/logs/export', LogExportV2.as_view()), + path('api/v2/logs/ip_list', LogsIPList.as_view()), + path('api/v2/logs/api_list', LogsApiList.as_view()), ]) # urlpatterns.extend(scaupload_urls) departured diff --git a/dongtai_web/views/log_api_list.py b/dongtai_web/views/log_api_list.py new file mode 100644 index 000000000..f73702c07 --- /dev/null +++ b/dongtai_web/views/log_api_list.py @@ -0,0 +1,46 @@ +import logging + +from django.http import JsonResponse +from django.utils.translation import gettext_lazy as _ +from drf_spectacular.utils import extend_schema +from rest_framework.request import Request + +from dongtai_common.endpoint import R, UserEndPoint +from dongtai_common.models.log import IastLog +from dongtai_common.models.user import User + +logger = logging.getLogger("dongtai-webapi") + + +class LogsApiList(UserEndPoint): + name = "api-v2-logs-api-list" + description = _("Log list") + + @extend_schema( + description="Get list of logs api.", + summary="Log API List", + tags=["Logs"], + ) + def get(self, request: Request) -> JsonResponse: + user: User = request.user # type: ignore + + try: + if user.is_system_admin(): + queryset = IastLog.objects.all() + elif user.is_talent_admin(): + users = self.get_auth_users(user) + user_ids = list(users.values_list("id", flat=True)) + queryset = IastLog.objects.filter(user_id__in=user_ids) + else: + queryset = IastLog.objects.filter(user=user) + + data = list( + queryset.values_list("url", flat=True) + .distinct() + .order_by("url") + ) + + return R.success(data=data) + except Exception as e: + logger.error(e, exc_info=True) + return R.success(data=list(), msg=_("failure")) diff --git a/dongtai_web/views/log_ip_list.py b/dongtai_web/views/log_ip_list.py new file mode 100644 index 000000000..fd2aef48e --- /dev/null +++ b/dongtai_web/views/log_ip_list.py @@ -0,0 +1,46 @@ +import logging + +from django.http import JsonResponse +from django.utils.translation import gettext_lazy as _ +from drf_spectacular.utils import extend_schema +from rest_framework.request import Request + +from dongtai_common.endpoint import R, UserEndPoint +from dongtai_common.models.log import IastLog +from dongtai_common.models.user import User + +logger = logging.getLogger("dongtai-webapi") + + +class LogsIPList(UserEndPoint): + name = "api-v2-logs-ip-list" + description = _("Log list") + + @extend_schema( + description="Get list of logs ip.", + summary="Log IP List", + tags=["Logs"], + ) + def get(self, request: Request) -> JsonResponse: + user: User = request.user # type: ignore + + try: + if user.is_system_admin(): + queryset = IastLog.objects.all() + elif user.is_talent_admin(): + users = self.get_auth_users(user) + user_ids = list(users.values_list("id", flat=True)) + queryset = IastLog.objects.filter(user_id__in=user_ids) + else: + queryset = IastLog.objects.filter(user=user) + + data = list( + queryset.values_list("access_ip", flat=True) + .distinct() + .order_by("access_ip") + ) + + return R.success(data=data) + except Exception as e: + logger.error(e, exc_info=True) + return R.success(data=list(), msg=_("failure")) diff --git a/dongtai_web/views/logs_v2.py b/dongtai_web/views/logs_v2.py index d68e0fe44..a6995b30c 100644 --- a/dongtai_web/views/logs_v2.py +++ b/dongtai_web/views/logs_v2.py @@ -1,8 +1,3 @@ -#!/usr/bin/env python -# -*- coding:utf-8 -*- -# author:sjh -# software: PyCharm -# project: webapi import logging from datetime import datetime @@ -26,6 +21,8 @@ class _LogsArgsSerializer(serializers.Serializer): page = serializers.IntegerField(min_value=1, default=1, help_text=_("Page index")) startTime = serializers.DateTimeField(default=None, help_text=_("The start time.")) endTime = serializers.DateTimeField(default=None, help_text=_("The end time.")) + ip = serializers.CharField(default=None, help_text=_("IP Address.")) + api = serializers.CharField(default=None, help_text=_("API URL.")) class _LogsDeleteSerializer(serializers.Serializer): @@ -70,6 +67,8 @@ def get(self, request: Request) -> JsonResponse: page_size: int = ser.validated_data.get("pageSize", 20) start_time: datetime | None = ser.validated_data.get("startTime", None) end_time: datetime | None = ser.validated_data.get("endTime", None) + ip: str | None = ser.validated_data.get("ip", None) + api: str | None = ser.validated_data.get("api", None) else: return R.failure(data="Can not validation data.") except ValidationError as e: @@ -90,7 +89,13 @@ def get(self, request: Request) -> JsonResponse: elif start_time is not None: queryset = queryset.filter(action_time__gte=start_time) elif end_time is not None: - queryset.filter(action_time__lte=end_time) + queryset = queryset.filter(action_time__lte=end_time) + + if ip is not None: + queryset = queryset.filter(access_ip=ip) + + if api is not None: + queryset = queryset.filter(url=api) # set cache key self.make_key(request) From a1e8a99ea87a9b8dad4bd6c5def629355ad02632 Mon Sep 17 00:00:00 2001 From: bidaya0 Date: Tue, 4 Jul 2023 14:31:07 +0800 Subject: [PATCH 040/161] fix: change init schema with filepath to filter log. --- dongtai_common/endpoint/__init__.py | 7 ++++--- dongtai_common/utils/init_schema.py | 7 +++++-- .../debug/management/commands/check_schema.py | 21 ++++++++++--------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/dongtai_common/endpoint/__init__.py b/dongtai_common/endpoint/__init__.py index 66029e11c..953f2bce4 100644 --- a/dongtai_common/endpoint/__init__.py +++ b/dongtai_common/endpoint/__init__.py @@ -117,14 +117,15 @@ def dispatch(self, request, *args, **kwargs): self.response = self.finalize_response(request, response, *args, **kwargs) - if self.request.user is not None and self.request.user.is_active and handler.__module__.startswith( - 'dongtai_web') and self.description is not None: + if self.request.user is not None and self.request.user.is_active and self.description is not None: try: method = self.request.method if method is None: raise ValueError("can not get request method") operate_method = method - path, _path_regex, schema = VIEW_CLASS_TO_SCHEMA[self.__class__][method] + path, _path_regex, schema, filepath = VIEW_CLASS_TO_SCHEMA[self.__class__][method] + if 'dongtai' in filepath and 'dongtai_protocol' not in filepath: + return self.response if schema is None: raise ValueError("can not get schema") tags: list[str] = schema["tags"] diff --git a/dongtai_common/utils/init_schema.py b/dongtai_common/utils/init_schema.py index daa736b1a..8c290c7a6 100644 --- a/dongtai_common/utils/init_schema.py +++ b/dongtai_common/utils/init_schema.py @@ -4,6 +4,7 @@ from drf_spectacular.generators import SchemaGenerator from drf_spectacular.drainage import add_trace_message +import inspect logger = logging.getLogger("django") VIEW_CLASS_TO_SCHEMA: dict[type, dict[str, tuple[str, str, dict | None]]] = {} @@ -24,6 +25,7 @@ def init_schema() -> None: for path, path_regex, method, view in endpoints: try: + filepath = inspect.getfile(view.__class__) with add_trace_message(getattr(view, "__class__", view).__name__): operation = view.schema.get_operation( path, path_regex, path_prefix, method, generator.registry @@ -33,6 +35,7 @@ def init_schema() -> None: path, path_regex, operation, + filepath, ) - except Exception: - logger.error(f"unable to get schema: view {view} of path {path}") + except Exception as e: + logger.error(f"unable to get schema: view {view} of path {path}", exc_info=e) diff --git a/test/debug/management/commands/check_schema.py b/test/debug/management/commands/check_schema.py index b3a53f2d2..3af8db52e 100644 --- a/test/debug/management/commands/check_schema.py +++ b/test/debug/management/commands/check_schema.py @@ -1,29 +1,30 @@ from dongtai_common.utils.init_schema import VIEW_CLASS_TO_SCHEMA from django.core.management.base import BaseCommand +from itertools import count class Command(BaseCommand): help = "Check API schema is complete" def handle(self, *args, **options): - count = 0 + n = count(0) for view, schema in VIEW_CLASS_TO_SCHEMA.items(): for method, schema in schema.items(): - _, _, schema = schema + _, _, schema, filepath = schema if schema is None: self.stdout.write(self.style.ERROR(f"No schema: {view}")) continue has_error = False - for schema_field in ("tags", "summary"): - if not schema.get(schema_field, None): - has_error = True - count += 1 - self.stdout.write( - self.style.ERROR( - f'[{count}]Miss "{schema_field}" schema: {method} {view}' - ) + schema_field_check = {schema_field: schema_field in schema for schema_field in ("tags", "summary")} + if not all(schema_field_check.values()): + has_error = True + missing_fields = [schema_field for schema_field, exists in schema_field_check.items()] + self.stdout.write( + self.style.ERROR( + f'[{next(n)}]Miss "{missing_fields}" schema: {method} {view} {filepath}' ) + ) if not has_error: self.stdout.write( self.style.SUCCESS( From def27df8a399a27bd027985158923120794f5ff0 Mon Sep 17 00:00:00 2001 From: bidaya0 Date: Tue, 4 Jul 2023 17:08:09 +0800 Subject: [PATCH 041/161] doc: add tags to api extend_schema. --- dongtai_common/utils/init_schema.py | 4 +- dongtai_protocol/views/agent_config.py | 12 ++- dongtai_protocol/views/agent_configv2.py | 4 +- dongtai_protocol/views/agent_limit.py | 7 +- dongtai_protocol/views/agent_register.py | 2 + dongtai_protocol/views/agent_update.py | 4 + dongtai_protocol/views/except_action.py | 7 +- dongtai_protocol/views/startuptime.py | 4 +- dongtai_web/dongtai_sca/views/newpackage.py | 2 + .../dongtai_sca/views/newpackagedetail.py | 6 +- .../dongtai_sca/views/newpackageprojects.py | 2 + .../views/newpackageprojectversions.py | 2 + .../dongtai_sca/views/newpackagesummary.py | 2 + .../dongtai_sca/views/newpackagevuldetail.py | 6 +- .../dongtai_sca/views/newpackagevullevel.py | 4 +- .../dongtai_sca/views/newpackagevuls.py | 2 + dongtai_web/threshold/agent_core_status.py | 22 +++++- dongtai_web/views/agent_status_update.py | 7 ++ dongtai_web/views/agent_summary.py | 7 +- dongtai_web/views/agents.py | 76 +++++++++---------- dongtai_web/views/captcha_create.py | 1 - .../views/engine_method_pool_detail.py | 6 +- dongtai_web/views/log_download.py | 10 +++ 23 files changed, 142 insertions(+), 57 deletions(-) diff --git a/dongtai_common/utils/init_schema.py b/dongtai_common/utils/init_schema.py index 8c290c7a6..cd0d71b8c 100644 --- a/dongtai_common/utils/init_schema.py +++ b/dongtai_common/utils/init_schema.py @@ -22,8 +22,10 @@ def init_schema() -> None: path_prefix = "/" if not path_prefix.startswith("^"): path_prefix = "^" + path_prefix # make sure regex only matches from the start - + from dongtai_common.endpoint import EndPoint for path, path_regex, method, view in endpoints: + if not issubclass(view.__class__, EndPoint): + continue try: filepath = inspect.getfile(view.__class__) with add_trace_message(getattr(view, "__class__", view).__name__): diff --git a/dongtai_protocol/views/agent_config.py b/dongtai_protocol/views/agent_config.py index a5daaebb5..113137a5f 100644 --- a/dongtai_protocol/views/agent_config.py +++ b/dongtai_protocol/views/agent_config.py @@ -18,6 +18,7 @@ from dongtai_common.utils.systemsettings import get_circuit_break from django.utils.translation import gettext_lazy as _ from result import Ok, Err, Result +from dongtai_common.utils.const import OPERATE_GET logger = logging.getLogger('dongtai.openapi') @@ -25,8 +26,10 @@ class AgentConfigView(OpenApiEndPoint): @extend_schema( + summary="agent配置", + tags=['Agent服务端交互协议', OPERATE_GET], + deprecated=True, description='Through agent_ Id get disaster recovery strategy', - responses=R, methods=['POST']) def post(self, request): try: @@ -59,7 +62,12 @@ def post(self, request): class AgentConfigv2View(OpenApiEndPoint): - + @extend_schema( + summary="agent配置", + tags=['Agent服务端交互协议', OPERATE_GET], + deprecated=True, + description='Through agent_ Id get disaster recovery strategy', + methods=['POST']) def post(self, request): try: param = parse_data(request.read()) diff --git a/dongtai_protocol/views/agent_configv2.py b/dongtai_protocol/views/agent_configv2.py index f4d8f5021..cbfbd84af 100644 --- a/dongtai_protocol/views/agent_configv2.py +++ b/dongtai_protocol/views/agent_configv2.py @@ -28,8 +28,8 @@ class AgentConfigAllinOneView(OpenApiEndPoint): @extend_schema_with_envcheck( [_AgentConfigArgsSerializer], - tags=['agent upload'], - description='Through agent_ Id get data gather strategy', + summary="agent配置", + tags=['Agent服务端交互协议'], methods=['GET']) def get(self, request): ser = _AgentConfigArgsSerializer(data=request.GET) diff --git a/dongtai_protocol/views/agent_limit.py b/dongtai_protocol/views/agent_limit.py index 25191970e..b2d3455f3 100644 --- a/dongtai_protocol/views/agent_limit.py +++ b/dongtai_protocol/views/agent_limit.py @@ -1,4 +1,3 @@ - from django.forms.models import model_to_dict import logging import time @@ -13,7 +12,11 @@ class LimitView(OpenApiEndPoint): - @extend_schema(description='Agent Limit', auth=[DongTaiAuth.TOKEN]) + + @extend_schema(summary="agent限制", + tags=['Agent服务端交互协议'], + deprecated=True, + methods=['GET']) def get(self, request): keys = ['cpu_limit'] profiles = IastProfile.objects.filter(key__in=keys).all() diff --git a/dongtai_protocol/views/agent_register.py b/dongtai_protocol/views/agent_register.py index e6407fe50..d5ff5286e 100644 --- a/dongtai_protocol/views/agent_register.py +++ b/dongtai_protocol/views/agent_register.py @@ -215,6 +215,8 @@ def register_server(agent_id, hostname, network, container_name, responses=[{ 204: None }], + tags=['Agent服务端交互协议'], + summary="agent注册", methods=['POST']) def post(self, request: Request): try: diff --git a/dongtai_protocol/views/agent_update.py b/dongtai_protocol/views/agent_update.py index 392e9c6c8..14b94dd4c 100644 --- a/dongtai_protocol/views/agent_update.py +++ b/dongtai_protocol/views/agent_update.py @@ -9,11 +9,15 @@ from django.utils.translation import gettext_lazy as _ from urllib.parse import urlparse, urlunparse from dongtai_web.views.project_add import is_ip +from dongtai_common.utils.const import OPERATE_PUT logger = logging.getLogger('dongtai.openapi') class AgentUpdateEndPoint(OpenApiEndPoint): @extend_schema( + summary="agent配置", + tags=['Agent服务端交互协议', OPERATE_PUT], + deprecated=True, description='Agent Update, Data is Gzip', responses=[ {204: None} diff --git a/dongtai_protocol/views/except_action.py b/dongtai_protocol/views/except_action.py index ffffcced0..51d643178 100644 --- a/dongtai_protocol/views/except_action.py +++ b/dongtai_protocol/views/except_action.py @@ -10,6 +10,7 @@ from urllib.parse import urlparse, urlunparse from dongtai_web.views.project_add import is_ip from rest_framework.viewsets import ViewSet +from dongtai_common.utils.const import OPERATE_PUT logger = logging.getLogger('dongtai.openapi') @@ -22,6 +23,8 @@ class AgentActionV2EndPoint(OpenApiEndPoint, ViewSet): responses=[{ 204: None }], + tags=['Agent服务端交互协议', OPERATE_PUT], + summary="agent实际状态", methods=['POST']) def actual_running_status(self, request): try: @@ -50,7 +53,9 @@ def actual_running_status(self, request): responses=[{ 204: None }], - methods=['POST']) + tags=[_('Agent服务端交互协议')], + summary="agent期望状态", + methods=['GET']) def except_running_status(self, request): if 'agentId' not in request.GET.keys(): return R.failure() diff --git a/dongtai_protocol/views/startuptime.py b/dongtai_protocol/views/startuptime.py index 975c5847d..df2d59670 100644 --- a/dongtai_protocol/views/startuptime.py +++ b/dongtai_protocol/views/startuptime.py @@ -27,7 +27,7 @@ class StartupTimeEndPoint(OpenApiEndPoint): name = "api-v1-startuptime" - @extend_schema(description='Agent Limit', auth=[DongTaiAuth.TOKEN]) + @extend_schema(tags=['Agent服务端交互协议'], summary="agent启动时间", deprecated=True) def post(self, request: Request): agent_id = request.data.get('agentId', None) startup_time = request.data.get('startupTime', None) @@ -43,7 +43,7 @@ def post(self, request: Request): class StartupTimeGzipEndPoint(StartupTimeEndPoint): name = "api-v1-startuptime" - @extend_schema(description='Agent Limit', auth=[DongTaiAuth.TOKEN]) + @extend_schema(tags=['Agent服务端交互协议'], summary="agent启动时间", deprecated=True) def post(self, request: Request): try: param = parse_data(request.read()) diff --git a/dongtai_web/dongtai_sca/views/newpackage.py b/dongtai_web/dongtai_sca/views/newpackage.py index 484e23541..cee9a2558 100644 --- a/dongtai_web/dongtai_sca/views/newpackage.py +++ b/dongtai_web/dongtai_sca/views/newpackage.py @@ -75,6 +75,8 @@ class Meta: class PackageList(UserEndPoint): @extend_schema_with_envcheck_v2(request=PackageListArgsSerializer, + tags=[_('Component')], + summary=_("Component List"), responses={200: _NewResponseSerializer}) def post(self, request): ser = PackageListArgsSerializer(data=request.data) diff --git a/dongtai_web/dongtai_sca/views/newpackagedetail.py b/dongtai_web/dongtai_sca/views/newpackagedetail.py index 1ec249b89..8945cb8ea 100644 --- a/dongtai_web/dongtai_sca/views/newpackagedetail.py +++ b/dongtai_web/dongtai_sca/views/newpackagedetail.py @@ -46,7 +46,11 @@ class PackageListArgsSerializer(serializers.Serializer): class PackageDetail(UserEndPoint): - @extend_schema_with_envcheck_v2(responses={200: _NewResponseSerializer}) + @extend_schema_with_envcheck_v2( + tags=[_('Component')], + summary=_("Component Detail"), + responses={200: _NewResponseSerializer}, + ) def get(self, request, language_id, package_name, package_version): asset = AssetV2Global.objects.filter( language_id=language_id, diff --git a/dongtai_web/dongtai_sca/views/newpackageprojects.py b/dongtai_web/dongtai_sca/views/newpackageprojects.py index 300186861..9dbd5c63d 100644 --- a/dongtai_web/dongtai_sca/views/newpackageprojects.py +++ b/dongtai_web/dongtai_sca/views/newpackageprojects.py @@ -59,6 +59,8 @@ class NewPackageRelationProject(UserEndPoint): @extend_schema_with_envcheck_v2( parameters=[RelationProjectArgsSerializer], + tags=[_('Component')], + summary="组件相关的项目版本", responses={200: FullRelationProjectResponseSerializer}) def get(self, request, language_id, package_name, package_version): ser = RelationProjectArgsSerializer(data=request.query_params) diff --git a/dongtai_web/dongtai_sca/views/newpackageprojectversions.py b/dongtai_web/dongtai_sca/views/newpackageprojectversions.py index 6f365a4e8..94a595fb4 100644 --- a/dongtai_web/dongtai_sca/views/newpackageprojectversions.py +++ b/dongtai_web/dongtai_sca/views/newpackageprojectversions.py @@ -55,6 +55,8 @@ class NewPackageRelationProjectVersion(UserEndPoint): @extend_schema_with_envcheck_v2( request=RelationProjectVersionArgsSerializer, + tags=[_('Component')], + summary="组件相关的项目版本", responses={200: FullRelationProjectVersionResponseSerializer}) def get(self, request, language_id, package_name, package_version, project_id): diff --git a/dongtai_web/dongtai_sca/views/newpackagesummary.py b/dongtai_web/dongtai_sca/views/newpackagesummary.py index b42ae6a34..f00a194ef 100644 --- a/dongtai_web/dongtai_sca/views/newpackagesummary.py +++ b/dongtai_web/dongtai_sca/views/newpackagesummary.py @@ -91,6 +91,8 @@ class NewPackageSummary(UserEndPoint): @extend_schema_with_envcheck_v2( parameters=[PackageSummaryArgsSerializer], + tags=[_('Component')], + summary="组件概况", responses={200: FullSummaryResponseSerializer}) def get(self, request): ser = PackageSummaryArgsSerializer(data=request.query_params) diff --git a/dongtai_web/dongtai_sca/views/newpackagevuldetail.py b/dongtai_web/dongtai_sca/views/newpackagevuldetail.py index 880e9cd7d..a759511dc 100644 --- a/dongtai_web/dongtai_sca/views/newpackagevuldetail.py +++ b/dongtai_web/dongtai_sca/views/newpackagevuldetail.py @@ -41,7 +41,11 @@ class PackageListArgsSerializer(serializers.Serializer): class PackageVulDetail(UserEndPoint): - @extend_schema_with_envcheck_v2(responses={200: _NewResponseSerializer}) + @extend_schema_with_envcheck_v2( + responses={200: _NewResponseSerializer}, + tags=[_('Component')], + summary="组件漏洞详情", + ) def get(self, request, vul_id): asset_vul = IastAssetVulV2.objects.filter(vul_id=vul_id).first() if asset_vul: diff --git a/dongtai_web/dongtai_sca/views/newpackagevullevel.py b/dongtai_web/dongtai_sca/views/newpackagevullevel.py index ea02ca27e..817dd4596 100644 --- a/dongtai_web/dongtai_sca/views/newpackagevullevel.py +++ b/dongtai_web/dongtai_sca/views/newpackagevullevel.py @@ -39,7 +39,9 @@ class Meta: class PackageVulLevels(UserEndPoint): - @extend_schema_with_envcheck_v2(responses={200: _NewResponseSerializer}) + @extend_schema_with_envcheck_v2(tags=[_('Component')], + summary="组件漏洞等级", + responses={200: _NewResponseSerializer}) def get(self, request): return R.success(data=[{ "id": level_id, diff --git a/dongtai_web/dongtai_sca/views/newpackagevuls.py b/dongtai_web/dongtai_sca/views/newpackagevuls.py index c0f26f2ac..462bcde00 100644 --- a/dongtai_web/dongtai_sca/views/newpackagevuls.py +++ b/dongtai_web/dongtai_sca/views/newpackagevuls.py @@ -37,6 +37,8 @@ class PackageVulsListArgsSerializer(serializers.Serializer): class NewPackageVuls(UserEndPoint): @extend_schema_with_envcheck_v2( + tags=[_('Component')], + summary="组件漏洞列表", parameters=[PackageVulsListArgsSerializer], responses={200: NewPackageVulSResponseSerializer}) def get(self, request, language_id, package_name, package_version): diff --git a/dongtai_web/threshold/agent_core_status.py b/dongtai_web/threshold/agent_core_status.py index 0f31bfdc5..fe44f2b9c 100644 --- a/dongtai_web/threshold/agent_core_status.py +++ b/dongtai_web/threshold/agent_core_status.py @@ -12,7 +12,7 @@ from dongtai_web.serializers.agent import AgentToggleArgsSerializer from dongtai_web.views import AGENT_STATUS from collections import defaultdict - +from dongtai_common.utils.const import OPERATE_PUT STATUS_MAPPING = defaultdict(lambda: 1, {3: 1, 4: 2}) @@ -34,10 +34,15 @@ class AgentCoreStatusUpdate(UserEndPoint): @extend_schema_with_envcheck( request=AgentToggleArgsSerializer, - tags=[_('Agent')], + tags=[ + _('Agent'), + OPERATE_PUT, + ], + deprecated=True, summary=_('Agent Status Update'), description=_("Control the running agent by specifying the id."), - response_schema=_ResponseSerializer) + response_schema=_ResponseSerializer, + ) def post(self, request): ser = AgentCoreStatusSerializer(data=request.data) if ser.is_valid(False): @@ -89,6 +94,17 @@ class AgentCoreStatusUpdateALL(UserEndPoint): name = "api-v1-agent-core-status-update" description = _("Suspend Agent") + @extend_schema_with_envcheck( + request=AgentToggleArgsSerializer, + tags=[ + _('Agent'), + OPERATE_PUT, + ], + deprecated=True, + summary="Agent 批量更新", + description=_("Control the running agent by specifying the id."), + response_schema=_ResponseSerializer, + ) def post(self, request): ser = AgentCoreStatusSerializer(data=request.data) department = request.user.get_relative_department() diff --git a/dongtai_web/views/agent_status_update.py b/dongtai_web/views/agent_status_update.py index 93cd06684..494317520 100644 --- a/dongtai_web/views/agent_status_update.py +++ b/dongtai_web/views/agent_status_update.py @@ -12,6 +12,7 @@ from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer +from dongtai_common.utils.const import OPERATE_PUT _ResponseSerializer = get_response_serializer( status_msg_keypair=(((201, _("Engine status was updated successfully.")), @@ -19,6 +20,12 @@ class AgentStatusUpdate(UserEndPoint): + @extend_schema_with_envcheck( + tags=[_('Agent'), OPERATE_PUT], + summary="探针状态修改", + deprecated=True, + response_schema=_ResponseSerializer, + ) def get(self, request): timestamp = int(time.time()) queryset = IastAgent.objects.filter(user=request.user) diff --git a/dongtai_web/views/agent_summary.py b/dongtai_web/views/agent_summary.py index 3a2cc7a43..c12a069cb 100644 --- a/dongtai_web/views/agent_summary.py +++ b/dongtai_web/views/agent_summary.py @@ -3,12 +3,17 @@ from dongtai_common.models.agent import IastAgent from dongtai_web.views.utils.commonstats import get_summary_by_agent_ids from dongtai_common.models.project_version import IastProjectVersion - +from drf_spectacular.utils import extend_schema class AgentSummary(UserEndPoint): name = "api-v1-agent-summary-" description = _("Item details - Summary") + @extend_schema( + tags=[_('Agent')], + summary="探针数量统计", + deprecated=True, + ) def get(self, request, pk): try: pk = int(pk) diff --git a/dongtai_web/views/agents.py b/dongtai_web/views/agents.py index f87902553..79dddcb3a 100644 --- a/dongtai_web/views/agents.py +++ b/dongtai_web/views/agents.py @@ -27,44 +27,6 @@ class AgentList(UserEndPoint): description = _("Agent list") SERVER_MAP = dict() - @extend_schema_with_envcheck( - [ - { - 'name': "page", - 'type': int, - 'default': 1, - 'required': False, - }, - { - 'name': "pageSize", - 'type': int, - 'default': 1, - 'required': False, - }, - { - 'name': "state", - 'type': int, - 'default': 1, - 'required': False, - }, - { - 'name': "token", - 'type': str, - 'required': False, - }, - { - 'name': "project_name", - 'type': str, - 'required': False, - }, - ], - tags=[_('Agent')], - summary=_('Agent List'), - description=_( - "Get a list containing Agent information according to conditions." - ), - response_schema=_ResponseSerializer, - ) def get_running_status(self, obj): mapping = defaultdict(str) mapping.update({1: _("Online"), 0: _("Offline")}) @@ -109,6 +71,44 @@ def parse_args(self, request): page_size = page_size if page_size < 50 else 50 return page, page_size, request.user + @extend_schema_with_envcheck( + [ + { + 'name': "page", + 'type': int, + 'default': 1, + 'required': False, + }, + { + 'name': "pageSize", + 'type': int, + 'default': 1, + 'required': False, + }, + { + 'name': "state", + 'type': int, + 'default': 1, + 'required': False, + }, + { + 'name': "token", + 'type': str, + 'required': False, + }, + { + 'name': "project_name", + 'type': str, + 'required': False, + }, + ], + tags=[_('Agent')], + summary=_('Agent List'), + description=_( + "Get a list containing Agent information according to conditions." + ), + response_schema=_ResponseSerializer, + ) def get(self, request): try: page = int(request.query_params.get('page', 1)) diff --git a/dongtai_web/views/captcha_create.py b/dongtai_web/views/captcha_create.py index 23601d823..3f6ca3fd2 100644 --- a/dongtai_web/views/captcha_create.py +++ b/dongtai_web/views/captcha_create.py @@ -8,7 +8,6 @@ from dongtai_common.endpoint import R from rest_framework.views import APIView - class CaptchaCreate(APIView): def get(self, request): hash_key = CaptchaStore.generate_key() diff --git a/dongtai_web/views/engine_method_pool_detail.py b/dongtai_web/views/engine_method_pool_detail.py index 29c81f38f..e8608099f 100644 --- a/dongtai_web/views/engine_method_pool_detail.py +++ b/dongtai_web/views/engine_method_pool_detail.py @@ -12,7 +12,7 @@ from dongtai_web.serializers.method_pool import MethodPoolListSerialize from django.utils.translation import gettext_lazy as _ -from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer +from drf_spectacular.utils import extend_schema logger = logging.getLogger('dongtai-webapi') @@ -21,6 +21,10 @@ class MethodPoolDetailProxy(AnonymousAndUserEndPoint): name = "api-engine-search" description = _("Engine - search data according to policy") + @extend_schema( + tags=[_('Method Pool')], + summary="方法调用链详情", + ) def post(self, request): """ :param request: diff --git a/dongtai_web/views/log_download.py b/dongtai_web/views/log_download.py index c20b62e48..8a2f4d9b2 100644 --- a/dongtai_web/views/log_download.py +++ b/dongtai_web/views/log_download.py @@ -18,6 +18,8 @@ from dongtai_conf.settings import TMP_COMMON_PATH from tempfile import NamedTemporaryFile from dongtai_common.endpoint import R +from drf_spectacular.utils import extend_schema +from django.utils.translation import gettext_lazy as _ logger = logging.getLogger('dongtai-webapi') @@ -54,6 +56,10 @@ def get_single(self, request, pk): return response return nothing_resp() + @extend_schema( + tags=[_('Agent')], + summary="探针日志批量下载", + ) def batch_task_add(self, request): mode = request.data.get('mode', 1) department = request.user.get_relative_department() @@ -71,6 +77,10 @@ def generate_zip_thread(): t1.start() return R.success() + @extend_schema( + tags=[_('Agent')], + summary="探针日志下载", + ) def batch_log_download(self, request, pk): try: a = int(pk) > 0 From a4e6b88972e0048dd5aeeeea70ee3b68c81223d5 Mon Sep 17 00:00:00 2001 From: st1020 Date: Tue, 4 Jul 2023 17:13:47 +0800 Subject: [PATCH 042/161] doc: add tags to api extend_schema --- dongtai_protocol/views/engine_auto_deploy.py | 6 ++++ dongtai_protocol/views/engine_download.py | 6 +++- dongtai_protocol/views/engine_heartbeat.py | 5 ++++ dongtai_protocol/views/engine_status.py | 10 +++++-- dongtai_protocol/views/health.py | 4 ++- dongtai_protocol/views/health_oss.py | 4 ++- dongtai_protocol/views/hook_profiles.py | 17 +++++++---- dongtai_protocol/views/properties.py | 6 ++++ dongtai_protocol/views/report_upload.py | 5 +++- dongtai_web/base/update_project_version.py | 5 ++++ .../dongtai_sca/views/asset_projects.py | 16 ++++++++++ dongtai_web/dongtai_sca/views/package.py | 1 + dongtai_web/dongtai_sca/views/package_vul.py | 16 ++++++++++ dongtai_web/threshold/config_setting.py | 16 ++++++++++ dongtai_web/versioncontrol/views.py | 6 ++++ dongtai_web/views/agents_v2.py | 9 ++++++ dongtai_web/views/details_id.py | 29 +++++++++++++++++++ .../views/engine_method_pool_detail.py | 5 ++++ dongtai_web/views/filereplace.py | 6 ++++ dongtai_web/views/log_clear.py | 6 ++++ dongtai_web/views/log_delete.py | 6 ++++ dongtai_web/views/log_export.py | 6 ++++ dongtai_web/views/logs.py | 6 ++++ dongtai_web/views/method_graph.py | 5 ++++ dongtai_web/views/system_info.py | 5 ++++ dongtai_web/views/user_detail.py | 5 ++++ dongtai_web/views/user_info.py | 5 ++++ dongtai_web/views/user_login.py | 5 ++++ dongtai_web/views/user_logout.py | 5 ++++ dongtai_web/views/user_passwrd.py | 5 ++++ dongtai_web/views/user_passwrd_reset.py | 5 ++++ dongtai_web/views/user_token.py | 9 ++++++ dongtai_web/views/version_update.py | 5 ++++ dongtai_web/views/vul_details.py | 5 ++++ dongtai_web/vul_log/vul_log_view.py | 5 ++++ 35 files changed, 248 insertions(+), 12 deletions(-) diff --git a/dongtai_protocol/views/engine_auto_deploy.py b/dongtai_protocol/views/engine_auto_deploy.py index 69551ef3d..979145dd4 100644 --- a/dongtai_protocol/views/engine_auto_deploy.py +++ b/dongtai_protocol/views/engine_auto_deploy.py @@ -9,6 +9,8 @@ from dongtai_common.endpoint import OpenApiEndPoint from django.http import StreamingHttpResponse from rest_framework.authtoken.models import Token +from drf_spectacular.utils import extend_schema +from django.utils.translation import gettext_lazy as _ logger = logging.getLogger("django") @@ -89,6 +91,10 @@ class AutoDeployEndPoint(OpenApiEndPoint): name = "download_iast_agent" description = "白帽子-下载IAST 自动部署脚本" + @extend_schema( + summary="下载 IAST 自动部署脚本", + tags=["Agent"] + ) def get(self, request): """ IAST下载 agent接口 diff --git a/dongtai_protocol/views/engine_download.py b/dongtai_protocol/views/engine_download.py index e2db70ed3..36aaeaf07 100644 --- a/dongtai_protocol/views/engine_download.py +++ b/dongtai_protocol/views/engine_download.py @@ -18,6 +18,8 @@ from dongtai_protocol.api_schema import DongTaiParameter from dongtai_protocol.utils import OssDownloader from dongtai_conf.settings import BUCKET_NAME_BASE_URL, VERSION +from django.utils.translation import gettext_lazy as _ + logger = logging.getLogger("dongtai.openapi") PACKAGE_NAME_LIST = ('dongtai-core', 'dongtai-spy', 'dongtai-api', @@ -39,7 +41,9 @@ class EngineDownloadEndPoint(OpenApiEndPoint): DongTaiParameter.ENGINE_NAME, ], responses=R, - methods=['GET'] + methods=['GET'], + summary="下载 Agent Engine", + tags=[_("Agent")] ) def get(self, request: Request): package_name = request.query_params.get('engineName') diff --git a/dongtai_protocol/views/engine_heartbeat.py b/dongtai_protocol/views/engine_heartbeat.py index 0c7b9e400..8017047f9 100644 --- a/dongtai_protocol/views/engine_heartbeat.py +++ b/dongtai_protocol/views/engine_heartbeat.py @@ -10,6 +10,7 @@ from dongtai_common.endpoint import OpenApiEndPoint, R from django.utils.translation import gettext_lazy as _ +from drf_spectacular.utils import extend_schema logger = logging.getLogger("dongtai.openapi") @@ -20,6 +21,10 @@ class EngineHeartBeatEndPoint(OpenApiEndPoint): name = "api-v1-report-upload" description = "agent上传报告" + @extend_schema( + summary="Agent 上传报告", + tags=["Agent"], + ) def post(self, request): """ IAST 检测引擎 agent接口 diff --git a/dongtai_protocol/views/engine_status.py b/dongtai_protocol/views/engine_status.py index b6099171f..b694b5e4f 100644 --- a/dongtai_protocol/views/engine_status.py +++ b/dongtai_protocol/views/engine_status.py @@ -12,7 +12,7 @@ from dongtai_common.endpoint import OpenApiEndPoint, R from drf_spectacular.utils import extend_schema from django.core.cache import cache - +from django.utils.translation import gettext_lazy as _ from dongtai_protocol.api_schema import DongTaiParameter logger = logging.getLogger("django") @@ -22,6 +22,10 @@ class EngineUpdateEndPoint(OpenApiEndPoint): name = "iast_engine_update_status_edit" description = "IAST 检测引擎更新状态修改接口" + @extend_schema( + summary="IAST 检测引擎更新状态修改接口", + tags=[_("Agent")], + ) def get(self, request, status=None): """ IAST 检测引擎 agent接口 @@ -62,7 +66,9 @@ class EngineAction(OpenApiEndPoint): DongTaiParameter.AGENT_NAME, ], responses=R, - methods=['GET'] + methods=['GET'], + summary="检查 Agent Engine 状态", + tags=[_("Agent")], ) def get(self, request): agent_id = request.query_params.get('agentId') diff --git a/dongtai_protocol/views/health.py b/dongtai_protocol/views/health.py index a5037bd29..e741336fd 100644 --- a/dongtai_protocol/views/health.py +++ b/dongtai_protocol/views/health.py @@ -86,7 +86,9 @@ class HealthView(UserEndPoint): @extend_schema( description='Check OpenAPI Service Status', responses=R, - methods=['GET'] + methods=['GET'], + summary="检查 OpenAPI 服务状态", + tags=["OpenAPI"] ) def get(self, request): oss_status, _ = checkossstatus() diff --git a/dongtai_protocol/views/health_oss.py b/dongtai_protocol/views/health_oss.py index bfcd8640c..ed054aae4 100644 --- a/dongtai_protocol/views/health_oss.py +++ b/dongtai_protocol/views/health_oss.py @@ -20,7 +20,9 @@ class OSSHealthView(UserEndPoint): @extend_schema( description='Check OSS Health', responses=R, - methods=['GET'] + methods=['GET'], + summary="Check OSS Health", + tags=["OSS"], ) def get(self, request): oss_status, _ = checkossstatus() diff --git a/dongtai_protocol/views/hook_profiles.py b/dongtai_protocol/views/hook_profiles.py index 265934c76..21f76a2b4 100644 --- a/dongtai_protocol/views/hook_profiles.py +++ b/dongtai_protocol/views/hook_profiles.py @@ -15,6 +15,7 @@ from django.db.models import (Prefetch, OuterRef, Subquery) # note: 当前依赖必须保留，否则无法通过hooktype反向查找策略 from dongtai_protocol.api_schema import DongTaiParameter +from django.utils.translation import gettext_lazy as _ from django.db.models import Q logger = logging.getLogger("django") @@ -112,12 +113,16 @@ def get_profiles(user=None, language_id=JAVA, full=False, system_only=False): key=lambda item: (item['value'], item['type'])) return profiles - @extend_schema(description='Pull Agent Engine Hook Rule', - parameters=[ - DongTaiParameter.LANGUAGE, - ], - responses=R, - methods=['GET']) + @extend_schema( + description='Pull Agent Engine Hook Rule', + parameters=[ + DongTaiParameter.LANGUAGE, + ], + responses=R, + methods=['GET'], + summary="Pull Agent Engine Hook Rule", + tags=[_("Agent")] + ) def get(self, request): user = request.user diff --git a/dongtai_protocol/views/properties.py b/dongtai_protocol/views/properties.py index f0dae4e29..334ce9baf 100644 --- a/dongtai_protocol/views/properties.py +++ b/dongtai_protocol/views/properties.py @@ -12,6 +12,8 @@ from dongtai_common.endpoint import OpenApiEndPoint, R from dongtai_protocol.serializers.agent_properties import AgentPropertiesSerialize +from drf_spectacular.utils import extend_schema +from django.utils.translation import gettext_lazy as _ logger = logging.getLogger("django") @@ -23,6 +25,10 @@ class PropertiesEndPoint(OpenApiEndPoint): name = "api-v1-properties" description = "获取属性配置" + @extend_schema( + summary="获取属性配置", + tags=[_("Profile")], + ) def get(self, request: Request): """ IAST下载 agent接口 diff --git a/dongtai_protocol/views/report_upload.py b/dongtai_protocol/views/report_upload.py index 40e2849e9..e269e8544 100644 --- a/dongtai_protocol/views/report_upload.py +++ b/dongtai_protocol/views/report_upload.py @@ -15,6 +15,7 @@ from rest_framework.views import APIView from django.http import JsonResponse from django.views.decorators.csrf import csrf_exempt +from django.utils.translation import gettext_lazy as _ logger = logging.getLogger('dongtai.openapi') @@ -29,7 +30,9 @@ class ReportUploadEndPoint(OpenApiEndPoint): DongTaiParameter.LANGUAGE, ], responses=R, - methods=['GET'] + methods=['GET'], + summary="Agent 上传报告", + tags=[_("Agent")], ) @csrf_exempt def post(self, request): diff --git a/dongtai_web/base/update_project_version.py b/dongtai_web/base/update_project_version.py index 4d1d561d0..67ef1b597 100644 --- a/dongtai_web/base/update_project_version.py +++ b/dongtai_web/base/update_project_version.py @@ -10,6 +10,7 @@ from dongtai_common.models.project import IastProject from dongtai_common.models.agent import IastAgent from django.utils.translation import gettext_lazy as _ +from drf_spectacular.utils import extend_schema logger = logging.getLogger("django") @@ -18,6 +19,10 @@ class UpdateProjectVersion(UserEndPoint): name = "api-v1-project-version-check" description = _("Detects and associates application version information") + @extend_schema( + summary=_("Detects and associates application version information"), + tags=["Project"], + ) def get(self, request): try: all_project = IastProject.objects.all() diff --git a/dongtai_web/dongtai_sca/views/asset_projects.py b/dongtai_web/dongtai_sca/views/asset_projects.py index 5dea6a1f9..39b55977f 100644 --- a/dongtai_web/dongtai_sca/views/asset_projects.py +++ b/dongtai_web/dongtai_sca/views/asset_projects.py @@ -19,6 +19,7 @@ from dongtai_web.serializers.sca import ScaSerializer from dongtai_web.dongtai_sca.serializers.asset_project import AssetProjectSerializer from typing import List, Optional, Dict +from drf_spectacular.utils import extend_schema logger = logging.getLogger(__name__) @@ -27,6 +28,11 @@ class AssetProjects(UserEndPoint): name = "api-v1-sca-projects" description = "" + @extend_schema( + deprecated=True, + summary="获取组件项目列表", + tags=["Project"], + ) def get(self, request, aggr_id): try: departments = request.user.get_relative_department() @@ -61,6 +67,11 @@ class AssetVulProjects(UserEndPoint): name = "api-v1-sca-vul-projects" description = "" + @extend_schema( + deprecated=True, + summary="获取组件漏洞项目列表", + tags=["Project"], + ) def get(self, request, vul_id): try: auth_users = self.get_auth_users(request.user) @@ -143,6 +154,11 @@ class ProjectsAssets(UserEndPoint): name = "api-v1-sca-vul-project-assets" description = "" + @extend_schema( + deprecated=True, + summary="获取项目组件列表", + tags=["Project"], + ) def get(self, request): try: diff --git a/dongtai_web/dongtai_sca/views/package.py b/dongtai_web/dongtai_sca/views/package.py index fc7c10fe6..8bc95a833 100644 --- a/dongtai_web/dongtai_sca/views/package.py +++ b/dongtai_web/dongtai_sca/views/package.py @@ -8,6 +8,7 @@ from django.forms.models import model_to_dict from dongtai_common.endpoint import R, AnonymousAndUserEndPoint, UserEndPoint from django.utils.translation import gettext_lazy as _ +from drf_spectacular.utils import extend_schema from dongtai_web.dongtai_sca.utils import get_asset_id_by_aggr_id diff --git a/dongtai_web/dongtai_sca/views/package_vul.py b/dongtai_web/dongtai_sca/views/package_vul.py index 8b75125a9..2e84e15b4 100644 --- a/dongtai_web/dongtai_sca/views/package_vul.py +++ b/dongtai_web/dongtai_sca/views/package_vul.py @@ -15,6 +15,7 @@ from dongtai_common.models.asset import Asset from collections import defaultdict import logging +from drf_spectacular.utils import extend_schema logger = logging.getLogger('dongtai-webapi') @@ -36,6 +37,11 @@ def find_fixed_versions(self, vul_package_id, ecosystem, name, version): fixed_versions.append(vul_package_range.fixed) return fixed_versions + @extend_schema( + deprecated=True, + summary="获取一个包内的漏洞列表", + tags=["Vulnerability"], + ) def get(self, request): filter_fields = ['hash', 'aql', 'ecosystem', 'name', 'version'] _filter = Package.objects.filter() @@ -112,6 +118,11 @@ class AssetPackageVulList(UserEndPoint): name = "api-v1-sca-package-vuls" description = "" + @extend_schema( + deprecated=True, + summary="获取组件中的包的漏洞列表", + tags=[_("Vulnerability")], + ) def get(self, request, aggr_id): auth_users = self.get_auth_users(request.user) departments = request.user.get_relative_department() @@ -195,6 +206,11 @@ class AssetPackageVulDetail(UserEndPoint): name = "api-v1-sca-package-vul-detail" description = "" + @extend_schema( + deprecated=True, + summary="获取组件中的包的漏洞详情", + tags=[_("Vulnerability")], + ) def get(self, request, vul_id): # 组件漏洞基础 数据读取 asset_vul = IastAssetVul.objects.filter(id=vul_id).first() diff --git a/dongtai_web/threshold/config_setting.py b/dongtai_web/threshold/config_setting.py index 49525743e..c8283a8d0 100644 --- a/dongtai_web/threshold/config_setting.py +++ b/dongtai_web/threshold/config_setting.py @@ -319,6 +319,10 @@ def list(self, request): obj_list.append(obj) return R.success(data=obj_list) + @extend_schema_with_envcheck( + summary=_('Update AgentThresholdConfig'), + description=_("Update AgentThresholdConfigV2"), + tags=[_('AgentThresholdConfigV2')]) def update(self, request, pk): ser = AgentConfigSettingV2Serializer(data=request.data) try: @@ -329,6 +333,10 @@ def update(self, request, pk): config_update(ser.data, pk) return R.success() + @extend_schema_with_envcheck( + summary=_('重置 AgentThresholdConfig'), + description=_("重置 AgentThresholdConfigV2"), + tags=[_('AgentThresholdConfigV2')]) def reset(self, request, pk): if IastCircuitConfig.objects.filter(pk=pk).exists(): config = IastCircuitConfig.objects.filter(pk=pk, ).first() @@ -355,10 +363,18 @@ def change_priority(self, request, pk): return R.success() return R.failure() + @extend_schema_with_envcheck( + summary=_('Delete AgentThresholdConfig'), + description=_("Delete AgentThresholdConfigV2"), + tags=[_('AgentThresholdConfigV2')]) def delete(self, request, pk): IastCircuitConfig.objects.filter(pk=pk).update(is_deleted=1) return R.success() + @extend_schema_with_envcheck( + summary=_('获取 AgentThresholdConfig 枚举'), + description=_("获取 AgentThresholdConfigV2 枚举"), + tags=[_('AgentThresholdConfigV2')]) def enum(self, request, enumname): able_to_search = (TargetType, MetricType, MetricGroup, TargetOperator, MetricOperator, DealType, SystemMetricType, diff --git a/dongtai_web/versioncontrol/views.py b/dongtai_web/versioncontrol/views.py index 1bb946082..02bd4dda6 100644 --- a/dongtai_web/versioncontrol/views.py +++ b/dongtai_web/versioncontrol/views.py @@ -3,6 +3,8 @@ from dongtai_common.endpoint import UserEndPoint from dongtai_common.models.version_control import VersionControl import json +from drf_spectacular.utils import extend_schema +from django.utils.translation import gettext_lazy as _ # Create your views here. COMPONENT_LIST = ("DongTai", "DongTai-agent-java", "DongTai-agent-python", @@ -10,6 +12,10 @@ class VersionListView(UserEndPoint): + @extend_schema( + summary="版本列表", + tags=[_("Version List")] + ) def get(self, request): component_datas = VersionControl.objects.filter( component_name__in=COMPONENT_LIST).all() diff --git a/dongtai_web/views/agents_v2.py b/dongtai_web/views/agents_v2.py index 149080654..791802d4c 100644 --- a/dongtai_web/views/agents_v2.py +++ b/dongtai_web/views/agents_v2.py @@ -30,6 +30,7 @@ from typing import Optional from time import time from itertools import groupby +from drf_spectacular.utils import extend_schema logger = logging.getLogger('dongtai-webapi') @@ -127,6 +128,10 @@ def pagenation_list(self, request): data = {'agents': queryset, "summary": summary} return R.success(data=data) + @extend_schema( + tags=[_('Agent')], + summary="获取 Agent 总结信息", + ) def summary(self, request): res = {} department = request.user.get_relative_department() @@ -144,6 +149,10 @@ def summary(self, request): return R.success(data=res) + @extend_schema( + tags=[_('Agent')], + summary="获取 Agent 状态", + ) def agent_stat(self, request): department = request.user.get_relative_department() try: diff --git a/dongtai_web/views/details_id.py b/dongtai_web/views/details_id.py index 55b2ef1cc..a668d800c 100644 --- a/dongtai_web/views/details_id.py +++ b/dongtai_web/views/details_id.py @@ -27,6 +27,7 @@ from rest_framework import serializers from rest_framework.serializers import ValidationError from dongtai_common.models.vulnerablity import IastVulnerabilityModel +from drf_spectacular.utils import extend_schema class IdsSerializer(serializers.Serializer): @@ -60,6 +61,13 @@ def get(self, request): class AgentListWithid(DetailListWithid): serializer = AgentSerializer + @extend_schema( + tags=[_('Agent')], + summary=_('Agent List with id'), + ) + def get(self, request): + return super().get(request) + def query(self, ids, request): agents = IastAgent.objects.filter(pk__in=ids, user__in=self.get_auth_users( @@ -80,6 +88,13 @@ def post(self, request): class ProjectListWithid(DetailListWithid): serializer = ProjectSerializer + @extend_schema( + tags=[_('Project')], + summary=_('Project List with id'), + ) + def get(self, request): + return super().get(request) + def query(self, ids, request): projects = IastProject.objects.filter(pk__in=ids, user__in=self.get_auth_users( @@ -100,6 +115,13 @@ def post(self, request): class ScaListWithid(DetailListWithid): serializer = ScaSerializer + @extend_schema( + tags=[_('Component')], + summary=_('Component List with id'), + ) + def get(self, request): + return super().get(request) + def query(self, ids, request): auth_users = self.get_auth_users(request.user) auth_agents = self.get_auth_agents(auth_users) @@ -120,6 +142,13 @@ def post(self, request): class VulsListWithid(DetailListWithid): serializer = VulSerializer + @extend_schema( + tags=[_('Vulnerability')], + summary=_('Vulnerability List with id'), + ) + def get(self, request): + return super().get(request) + def query(self, ids, request): auth_users = self.get_auth_users(request.user) auth_agents = self.get_auth_agents(auth_users) diff --git a/dongtai_web/views/engine_method_pool_detail.py b/dongtai_web/views/engine_method_pool_detail.py index 29c81f38f..96c8e0955 100644 --- a/dongtai_web/views/engine_method_pool_detail.py +++ b/dongtai_web/views/engine_method_pool_detail.py @@ -12,6 +12,7 @@ from dongtai_web.serializers.method_pool import MethodPoolListSerialize from django.utils.translation import gettext_lazy as _ +from drf_spectacular.utils import extend_schema from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer logger = logging.getLogger('dongtai-webapi') @@ -21,6 +22,10 @@ class MethodPoolDetailProxy(AnonymousAndUserEndPoint): name = "api-engine-search" description = _("Engine - search data according to policy") + @extend_schema( + summary=_("Engine - search data according to policy"), + tags=["Engine"], + ) def post(self, request): """ :param request: diff --git a/dongtai_web/views/filereplace.py b/dongtai_web/views/filereplace.py index 535d558bf..cca0e7a28 100644 --- a/dongtai_web/views/filereplace.py +++ b/dongtai_web/views/filereplace.py @@ -13,6 +13,8 @@ import os from dongtai_conf.settings import MEDIA_ROOT from rest_framework.parsers import FileUploadParser +from drf_spectacular.utils import extend_schema + logger = logging.getLogger('dongtai-webapi') FILES_ALLOWED_MODIFIY = ("logo.png", "logo_en.png", "favicon.ico") @@ -40,6 +42,10 @@ class FileReplace(TalentAdminEndPoint): + @extend_schema( + summary="替换文件", + tags=[_("Profile")] + ) def post(self, request, filename: str): if filename not in FILES_ALLOWED_MODIFIY: return R.failure(msg=_( diff --git a/dongtai_web/views/log_clear.py b/dongtai_web/views/log_clear.py index ee0871b5b..7b620dd21 100644 --- a/dongtai_web/views/log_clear.py +++ b/dongtai_web/views/log_clear.py @@ -8,12 +8,18 @@ from dongtai_common.endpoint import UserEndPoint, R from django.utils.translation import gettext_lazy as _ import datetime +from drf_spectacular.utils import extend_schema class LogClear(UserEndPoint): name = 'api-v1-log-clear' description = _('Log clear') + @extend_schema( + deprecated=True, + summary=_('Log clear'), + tags=[_("Logs")], + ) def get(self, request): user = request.user now = datetime.datetime.now() diff --git a/dongtai_web/views/log_delete.py b/dongtai_web/views/log_delete.py index d0a325565..6d72c80b7 100644 --- a/dongtai_web/views/log_delete.py +++ b/dongtai_web/views/log_delete.py @@ -7,12 +7,18 @@ from django.contrib.admin.models import LogEntry from dongtai_common.endpoint import UserEndPoint, R from django.utils.translation import gettext_lazy as _ +from drf_spectacular.utils import extend_schema class LogDelete(UserEndPoint): name = 'api-v1-log-delete' description = _('Log delete') + @extend_schema( + deprecated=True, + summary=_('Log delete'), + tags=[_("Logs")] + ) def post(self, request): ids = request.data.get('ids') if ids: diff --git a/dongtai_web/views/log_export.py b/dongtai_web/views/log_export.py index 81c1b6f2e..4f1ec732f 100644 --- a/dongtai_web/views/log_export.py +++ b/dongtai_web/views/log_export.py @@ -11,6 +11,7 @@ from django.utils.translation import gettext_lazy as _ from dongtai_common.endpoint import UserEndPoint from dongtai_common.endpoint import R +from drf_spectacular.utils import extend_schema class LogResurce(resources.ModelResource): @@ -41,6 +42,11 @@ def attachment_response(export_data, filename='download.xls', content_type='appl response['Content-Disposition'] = "attachment; filename*=utf-8''{}".format(escape_uri_path(filename)) return response + @extend_schema( + deprecated=True, + summary="日志导出", + tags=[_("Logs")] + ) def get(self, request): ids = request.query_params.get('ids') if ids: diff --git a/dongtai_web/views/logs.py b/dongtai_web/views/logs.py index ed74bfd97..004c6045c 100644 --- a/dongtai_web/views/logs.py +++ b/dongtai_web/views/logs.py @@ -8,6 +8,7 @@ from dongtai_common.endpoint import UserEndPoint, R from django.utils.translation import gettext_lazy as _ from django.core.cache import cache +from drf_spectacular.utils import extend_schema logger = logging.getLogger('dongtai-webapi') @@ -41,6 +42,11 @@ def parse_args(self, request): page_size = page_size if page_size < 50 else 50 return page, page_size, request.user + @extend_schema( + deprecated=True, + summary="获取日志列表", + tags=[_("Logs")] + ) def get(self, request): try: page, page_size, user = self.parse_args(request) diff --git a/dongtai_web/views/method_graph.py b/dongtai_web/views/method_graph.py index c02ec595d..8f8a39dd0 100644 --- a/dongtai_web/views/method_graph.py +++ b/dongtai_web/views/method_graph.py @@ -14,11 +14,16 @@ from dongtai_common.utils import const from dongtai_common.utils.validate import Validate from django.utils.translation import gettext_lazy as _ +from drf_spectacular.utils import extend_schema logger = logging.getLogger('dongtai-webapi') class MethodGraph(AnonymousAndUserEndPoint): + @extend_schema( + summary="调用链图", + tags=["Method Pool"] + ) def get(self, request): try: method_pool_id = int(request.query_params.get('method_pool_id')) diff --git a/dongtai_web/views/system_info.py b/dongtai_web/views/system_info.py index 0fbce9be3..3e44b1ea1 100644 --- a/dongtai_web/views/system_info.py +++ b/dongtai_web/views/system_info.py @@ -5,6 +5,7 @@ # project: lingzhi-webapi from dongtai_common.endpoint import R +from drf_spectacular.utils import extend_schema from django.utils.translation import gettext_lazy as _ from dongtai_common.endpoint import TalentAdminEndPoint @@ -13,5 +14,9 @@ class SystemInfo(TalentAdminEndPoint): name = "api-v1-system-info" description = _("API - System Information Page") + @extend_schema( + summary=_("API - System Information Page"), + tags=[_("System")] + ) def get(self, request): return R.success() diff --git a/dongtai_web/views/user_detail.py b/dongtai_web/views/user_detail.py index 898d6fcd7..ca1779a40 100644 --- a/dongtai_web/views/user_detail.py +++ b/dongtai_web/views/user_detail.py @@ -9,9 +9,14 @@ from dongtai_common.endpoint import TalentAdminEndPoint from dongtai_common.models import User from django.utils.translation import gettext_lazy as _ +from drf_spectacular.utils import extend_schema class UserDetailEndPoint(TalentAdminEndPoint): + @extend_schema( + summary=_("User Detail"), + tags=[_("User")], + ) def get(self, request, user_id): try: user = User.objects.filter(id=user_id).first() diff --git a/dongtai_web/views/user_info.py b/dongtai_web/views/user_info.py index d50626042..3655301b3 100644 --- a/dongtai_web/views/user_info.py +++ b/dongtai_web/views/user_info.py @@ -12,6 +12,7 @@ from dongtai_common.endpoint import UserEndPoint from django.utils.translation import gettext_lazy as _ from dongtai_conf.settings import SCA_SETUP +from drf_spectacular.utils import extend_schema logger = logging.getLogger("django") @@ -20,6 +21,10 @@ class UserInfoEndpoint(UserEndPoint): name = "api-v1-user-info" description = _("User Info") + @extend_schema( + summary=_("User Info"), + tags=[_("User")], + ) def get(self, request): user = request.user group = Group.objects.filter(user=user).order_by("-id").first() diff --git a/dongtai_web/views/user_login.py b/dongtai_web/views/user_login.py index e76aa3fab..20614e432 100644 --- a/dongtai_web/views/user_login.py +++ b/dongtai_web/views/user_login.py @@ -10,6 +10,7 @@ from django.utils.translation import gettext_lazy as _ from dongtai_web.projecttemplate.update_department_data import update_department_data import time +from drf_spectacular.utils import extend_schema logger = logging.getLogger("dongtai-webapi") @@ -20,6 +21,10 @@ class UserLogin(UserEndPoint): name = "user_views_login" description = _("User login") + @extend_schema( + summary=_("User login"), + tags=[_("User")], + ) def post(self, request): """{ 'username': "", diff --git a/dongtai_web/views/user_logout.py b/dongtai_web/views/user_logout.py index 1c3157097..76e083820 100644 --- a/dongtai_web/views/user_logout.py +++ b/dongtai_web/views/user_logout.py @@ -10,6 +10,7 @@ from django.contrib.auth import logout from django.http import JsonResponse from dongtai_common.endpoint import AnonymousAuthEndPoint +from drf_spectacular.utils import extend_schema from django.utils.translation import gettext_lazy as _ from dongtai_conf import settings @@ -21,6 +22,10 @@ class UserLogout(AnonymousAuthEndPoint): name = "api-v1-user-logout" description = _("Sign out") + @extend_schema( + summary=_("Sign out"), + tags=[_("User")], + ) def get(self, request): logout(request) response = JsonResponse({ diff --git a/dongtai_web/views/user_passwrd.py b/dongtai_web/views/user_passwrd.py index 0305cbee8..ee43f7186 100644 --- a/dongtai_web/views/user_passwrd.py +++ b/dongtai_web/views/user_passwrd.py @@ -10,6 +10,7 @@ from dongtai_common.endpoint import R from dongtai_common.endpoint import UserEndPoint from django.utils.translation import gettext_lazy as _ +from drf_spectacular.utils import extend_schema logger = logging.getLogger("dongtai-webapi") @@ -18,6 +19,10 @@ class UserPassword(UserEndPoint): name = "api-v1-user-password" description = _("Change Password") + @extend_schema( + summary=_("Change Password"), + tags=[_("User")], + ) def post(self, request): user = request.user try: diff --git a/dongtai_web/views/user_passwrd_reset.py b/dongtai_web/views/user_passwrd_reset.py index 24bd4f017..fe97b8dcc 100644 --- a/dongtai_web/views/user_passwrd_reset.py +++ b/dongtai_web/views/user_passwrd_reset.py @@ -9,6 +9,7 @@ from dongtai_common.endpoint import R from dongtai_common.endpoint import TalentAdminEndPoint +from drf_spectacular.utils import extend_schema from django.utils.translation import gettext_lazy as _ logger = logging.getLogger("dongtai-webapi") @@ -18,6 +19,10 @@ class UserPasswordReset(TalentAdminEndPoint): name = "api-v1-user-password-reset" description = _("Reset Password") + @extend_schema( + summary=_("Reset Password"), + tags=[_("User")], + ) def post(self, request): try: user_id = request.data.get('userId') diff --git a/dongtai_web/views/user_token.py b/dongtai_web/views/user_token.py index 11e9724c7..6d84041fa 100644 --- a/dongtai_web/views/user_token.py +++ b/dongtai_web/views/user_token.py @@ -9,6 +9,7 @@ from dongtai_common.endpoint import R from dongtai_common.endpoint import UserEndPoint from rest_framework.authtoken.models import Token +from drf_spectacular.utils import extend_schema from django.utils.translation import gettext_lazy as _ from dongtai_web.projecttemplate.update_department_data import update_department_data @@ -19,6 +20,10 @@ class UserToken(UserEndPoint): name = "iast-v1-user-token" description = _("Get OpenAPI token") + @extend_schema( + summary=_("Get OpenAPI token"), + tags=[_("User")], + ) def get(self, request): token, success = Token.objects.get_or_create(user=request.user) @@ -29,6 +34,10 @@ class UserDepartmentToken(UserEndPoint): name = "iast-v1-user-department-token" description = _("Get Department Deploy token") + @extend_schema( + summary=_("Get Department Deploy token"), + tags=[_("User")], + ) def get(self, request): departments = request.user.get_relative_department() department = request.user.get_department() diff --git a/dongtai_web/views/version_update.py b/dongtai_web/views/version_update.py index 20dcc0696..c30476fd2 100644 --- a/dongtai_web/views/version_update.py +++ b/dongtai_web/views/version_update.py @@ -9,11 +9,16 @@ from dongtai_common.models.agent_method_pool import MethodPool from dongtai_common.models.profile import IastProfile from django.utils.translation import gettext_lazy as _ +from drf_spectacular.utils import extend_schema logger = logging.getLogger('dongtai-webapi') class MethodPoolVersionUpdate(TalentAdminEndPoint): + @extend_schema( + summary="版本更新", + tags=["Version Update"], + ) def get(self, request): profile_model = IastProfile.objects.filter(key='enable_update').first() if profile_model is None or profile_model.value != 'TRUE': diff --git a/dongtai_web/views/vul_details.py b/dongtai_web/views/vul_details.py index b3b69c044..7b7e9671e 100644 --- a/dongtai_web/views/vul_details.py +++ b/dongtai_web/views/vul_details.py @@ -23,6 +23,7 @@ from django.db.models.base import ObjectDoesNotExist from dongtai_common.utils.stack_recognize import stacks_convert from dongtai_common.models.recognize_rule import IastRecognizeRule, RuleTypeChoices +from drf_spectacular.utils import extend_schema logger = logging.getLogger('dongtai-webapi') @@ -480,6 +481,10 @@ def get_graph_and_headers(self, data): }] return res + @extend_schema( + summary="获取漏洞详情", + tags=["Vulnerability"], + ) def get(self, request, id, ): self.vul_id = id self.departments = request.user.get_relative_department() diff --git a/dongtai_web/vul_log/vul_log_view.py b/dongtai_web/vul_log/vul_log_view.py index f7cc1eb4f..772ba7539 100644 --- a/dongtai_web/vul_log/vul_log_view.py +++ b/dongtai_web/vul_log/vul_log_view.py @@ -15,12 +15,17 @@ from django.utils.translation import gettext_lazy as _ from rest_framework import viewsets from dongtai_web.common import VulType +from drf_spectacular.utils import extend_schema class VulLogViewSet(UserEndPoint, viewsets.ViewSet): name = "api-v1-vul-log" description = _("vul-log") + @extend_schema( + tags=[_("Vulnerability")], + summary="漏洞日志", + ) def list(self, request, vul_id): data = [] auth_users = self.get_auth_users(request.user) From 068539952482f6b7986dc1693952acbb298ec6a5 Mon Sep 17 00:00:00 2001 From: bidaya0 Date: Tue, 4 Jul 2023 17:27:39 +0800 Subject: [PATCH 043/161] doc: add tags to api extend_schema. --- dongtai_protocol/views/engine_auto_deploy.py | 3 ++- dongtai_protocol/views/engine_download.py | 2 +- dongtai_protocol/views/engine_status.py | 5 +++-- dongtai_protocol/views/hook_profiles.py | 2 +- dongtai_protocol/views/report_upload.py | 2 +- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/dongtai_protocol/views/engine_auto_deploy.py b/dongtai_protocol/views/engine_auto_deploy.py index 979145dd4..5349b4467 100644 --- a/dongtai_protocol/views/engine_auto_deploy.py +++ b/dongtai_protocol/views/engine_auto_deploy.py @@ -93,7 +93,8 @@ class AutoDeployEndPoint(OpenApiEndPoint): @extend_schema( summary="下载 IAST 自动部署脚本", - tags=["Agent"] + tags=["Agent"], + deprecated=True, ) def get(self, request): """ diff --git a/dongtai_protocol/views/engine_download.py b/dongtai_protocol/views/engine_download.py index 36aaeaf07..9523e9dfd 100644 --- a/dongtai_protocol/views/engine_download.py +++ b/dongtai_protocol/views/engine_download.py @@ -43,7 +43,7 @@ class EngineDownloadEndPoint(OpenApiEndPoint): responses=R, methods=['GET'], summary="下载 Agent Engine", - tags=[_("Agent")] + tags=['Agent服务端交互协议'], ) def get(self, request: Request): package_name = request.query_params.get('engineName') diff --git a/dongtai_protocol/views/engine_status.py b/dongtai_protocol/views/engine_status.py index b694b5e4f..07d99ae5c 100644 --- a/dongtai_protocol/views/engine_status.py +++ b/dongtai_protocol/views/engine_status.py @@ -24,7 +24,7 @@ class EngineUpdateEndPoint(OpenApiEndPoint): @extend_schema( summary="IAST 检测引擎更新状态修改接口", - tags=[_("Agent")], + tags=['Agent服务端交互协议'], ) def get(self, request, status=None): """ @@ -68,7 +68,8 @@ class EngineAction(OpenApiEndPoint): responses=R, methods=['GET'], summary="检查 Agent Engine 状态", - tags=[_("Agent")], + tags=['Agent服务端交互协议'], + deprecated=True, ) def get(self, request): agent_id = request.query_params.get('agentId') diff --git a/dongtai_protocol/views/hook_profiles.py b/dongtai_protocol/views/hook_profiles.py index 21f76a2b4..2f5e2b1f9 100644 --- a/dongtai_protocol/views/hook_profiles.py +++ b/dongtai_protocol/views/hook_profiles.py @@ -121,7 +121,7 @@ def get_profiles(user=None, language_id=JAVA, full=False, system_only=False): responses=R, methods=['GET'], summary="Pull Agent Engine Hook Rule", - tags=[_("Agent")] + tags=['Agent服务端交互协议'], ) def get(self, request): user = request.user diff --git a/dongtai_protocol/views/report_upload.py b/dongtai_protocol/views/report_upload.py index e269e8544..f021c2530 100644 --- a/dongtai_protocol/views/report_upload.py +++ b/dongtai_protocol/views/report_upload.py @@ -32,7 +32,7 @@ class ReportUploadEndPoint(OpenApiEndPoint): responses=R, methods=['GET'], summary="Agent 上传报告", - tags=[_("Agent")], + tags=['Agent服务端交互协议'], ) @csrf_exempt def post(self, request): From b7da1cf71b2696ba6f349b6141f4c6d1ab113f50 Mon Sep 17 00:00:00 2001 From: bidaya0 Date: Tue, 4 Jul 2023 18:00:15 +0800 Subject: [PATCH 044/161] doc: add tags to api extend_schema. --- dongtai_protocol/views/report_upload.py | 7 ---- dongtai_web/dongtai_sca/views/package.py | 10 +++++ dongtai_web/threshold/config_setting.py | 9 +++++ dongtai_web/views/vul_request_replay.py | 38 +++++++++++-------- static/i18n/views/setlang.py | 14 ++++--- .../debug/management/commands/check_schema.py | 5 ++- 6 files changed, 54 insertions(+), 29 deletions(-) diff --git a/dongtai_protocol/views/report_upload.py b/dongtai_protocol/views/report_upload.py index f021c2530..a40eaa38f 100644 --- a/dongtai_protocol/views/report_upload.py +++ b/dongtai_protocol/views/report_upload.py @@ -25,16 +25,9 @@ class ReportUploadEndPoint(OpenApiEndPoint): description = "agent上传报告" @extend_schema( - description='Pull Agent Engine Hook Rule', - parameters=[ - DongTaiParameter.LANGUAGE, - ], - responses=R, - methods=['GET'], summary="Agent 上传报告", tags=['Agent服务端交互协议'], ) - @csrf_exempt def post(self, request): try: report = parse_data(request.read()) diff --git a/dongtai_web/dongtai_sca/views/package.py b/dongtai_web/dongtai_sca/views/package.py index 8bc95a833..bf1dc510f 100644 --- a/dongtai_web/dongtai_sca/views/package.py +++ b/dongtai_web/dongtai_sca/views/package.py @@ -17,6 +17,11 @@ class PackageList(AnonymousAndUserEndPoint): + @extend_schema( + tags=[_('Component')], + summary="组件列表", + deprecated=True, + ) def get(self, request): filter_fields = ['hash', 'aql', 'ecosystem', 'name', 'version'] _filter = Package.objects.filter().order_by("-updated_at") @@ -54,6 +59,11 @@ class AssetAggrDetailAssetIds(UserEndPoint): name = "api-v1-sca-aggr-assets" description = "" + @extend_schema( + tags=[_('Component')], + summary="组件详情", + deprecated=True, + ) def get(self, request, aggr_id): try: auth_users = self.get_auth_users(request.user) diff --git a/dongtai_web/threshold/config_setting.py b/dongtai_web/threshold/config_setting.py index c8283a8d0..7a5a7a437 100644 --- a/dongtai_web/threshold/config_setting.py +++ b/dongtai_web/threshold/config_setting.py @@ -291,6 +291,9 @@ def create(self, request): config_create(ser.data, request.user) return R.success() + @extend_schema_with_envcheck( + summary=_('AgentThresholdConfig Detail'), + tags=[_('AgentThresholdConfigV2')]) def retrieve(self, request, pk): obj = IastCircuitConfig.objects.filter(pk=pk, is_deleted=0).values().first() @@ -304,6 +307,9 @@ def retrieve(self, request, pk): circuit_config_id=pk).values().all()) return R.success(data=obj) + @extend_schema_with_envcheck( + summary=_('AgentThresholdConfig List'), + tags=[_('AgentThresholdConfigV2')]) def list(self, request): # page = request.query_params.get('page', 1) # page_size = request.query_params.get("page_size", 10) @@ -388,6 +394,9 @@ def enum(self, request, enumname): return R.success(data=convert_choices_to_value_dict( able_to_search_dict.get(enumname))) + @extend_schema_with_envcheck( + summary=_('获取 AgentThresholdConfig 所有枚举'), + tags=[_('AgentThresholdConfigV2')]) def enumall(self, request): able_to_search = (TargetType, MetricType, MetricGroup, TargetOperator, MetricOperator, DealType, SystemMetricType, diff --git a/dongtai_web/views/vul_request_replay.py b/dongtai_web/views/vul_request_replay.py index aec1cb057..919052daa 100644 --- a/dongtai_web/views/vul_request_replay.py +++ b/dongtai_web/views/vul_request_replay.py @@ -167,12 +167,16 @@ def send_request_to_replay_queue(relation_id, agent_id, replay_request, return replay_queue.id @extend_schema_with_envcheck( - [], { + [], + { 'methodPoolId': 'int', 'replayRequest': 'str', 'agent_id': 'int', 'replay_type': 'int' - }) + }, + tags=['重放'], + summary="请求包重放", + ) def post(self, request): """ :param request:{ @@ -255,19 +259,23 @@ def parse_response(header, body): e))) return '{header}\n\n{body}'.format(header=_data, body=body) - @extend_schema_with_envcheck([ - { - 'name': "replayid", - 'type': int, - 'required': True, - }, - { - 'name': "replay_type", - 'type': int, - 'required': False, - 'description': "available options are (2,3)", - }, - ]) + @extend_schema_with_envcheck( + [ + { + 'name': "replayid", + 'type': int, + 'required': True, + }, + { + 'name': "replay_type", + 'type': int, + 'required': False, + 'description': "available options are (2,3)", + }, + ], + tags=['重放'], + summary="获取请求包重放详情", + ) def get(self, request): replay_id = request.query_params.get('replayId') # auth_agents = self.get_auth_agents_with_user(request.user) diff --git a/static/i18n/views/setlang.py b/static/i18n/views/setlang.py index c167cbdf8..5b95f7449 100644 --- a/static/i18n/views/setlang.py +++ b/static/i18n/views/setlang.py @@ -18,11 +18,15 @@ class LanguageSetting(AnonymousAndUserEndPoint): - @extend_schema_with_envcheck([{ - 'name': LANGUAGE_QUERY_PARAMETER, - 'type': str, - 'description': 'The options are (en,zh)' - }]) + @extend_schema_with_envcheck( + [{ + 'name': LANGUAGE_QUERY_PARAMETER, + 'type': str, + 'description': 'The options are (en,zh)' + }], + tags=['i18n'], + summary="切换语言", + ) def get(self, request): lang_code = request.GET.get(LANGUAGE_QUERY_PARAMETER) if lang_code not in ALLOWED_LANG_CODE: diff --git a/test/debug/management/commands/check_schema.py b/test/debug/management/commands/check_schema.py index 3af8db52e..ad5a09bfe 100644 --- a/test/debug/management/commands/check_schema.py +++ b/test/debug/management/commands/check_schema.py @@ -11,18 +11,19 @@ def handle(self, *args, **options): n = count(0) for view, schema in VIEW_CLASS_TO_SCHEMA.items(): for method, schema in schema.items(): - _, _, schema, filepath = schema + path, path_regex, schema, filepath = schema if schema is None: self.stdout.write(self.style.ERROR(f"No schema: {view}")) continue has_error = False + schema_field_check = {schema_field: schema_field in schema for schema_field in ("tags", "summary")} if not all(schema_field_check.values()): has_error = True missing_fields = [schema_field for schema_field, exists in schema_field_check.items()] self.stdout.write( self.style.ERROR( - f'[{next(n)}]Miss "{missing_fields}" schema: {method} {view} {filepath}' + f'[{next(n)}]Miss "{missing_fields}" schema: {method} {view} {filepath} {path} {path_regex}' ) ) if not has_error: From 1e4a3fd9b21b08ffd853d34a9c3213c636bfcd77 Mon Sep 17 00:00:00 2001 From: st1020 Date: Tue, 4 Jul 2023 18:00:33 +0800 Subject: [PATCH 045/161] doc: add tags to api extend_schema --- .../vul_recheck_payload.py | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/dongtai_web/vul_recheck_payload/vul_recheck_payload.py b/dongtai_web/vul_recheck_payload/vul_recheck_payload.py index a441b86cd..d3f555437 100644 --- a/dongtai_web/vul_recheck_payload/vul_recheck_payload.py +++ b/dongtai_web/vul_recheck_payload/vul_recheck_payload.py @@ -11,7 +11,8 @@ from rest_framework import serializers from django.db.models import Q from django.forms.models import model_to_dict - +from django.utils.translation import gettext as _ +from drf_spectacular.utils import extend_schema def get_or_none(classmodel, function, **kwargs): try: @@ -57,6 +58,10 @@ class VulReCheckPayloadViewSet(UserEndPoint, viewsets.ViewSet): name = "api-v1-vul-recheck-payload" description = _("config recheck payload V2") + @extend_schema( + summary="漏洞验证", + tags=[_("Vulnerability")], + ) def create(self, request): ser = IastVulRecheckPayloadSerializer(data=request.data) try: @@ -67,6 +72,10 @@ def create(self, request): vul_recheck_payload_create(ser.data, request.user.id) return R.success() + @extend_schema( + summary="漏洞验证", + tags=[_("Vulnerability")], + ) def retrieve(self, request, pk): obj = get_or_none( IastVulRecheckPayload, @@ -78,6 +87,10 @@ def retrieve(self, request, pk): return R.failure() return R.success(data=obj) + @extend_schema( + summary="漏洞验证列表", + tags=[_("Vulnerability")], + ) def list(self, request): ser = IastVulRecheckPayloadListSerializer(data=request.query_params) try: @@ -106,6 +119,10 @@ def list(self, request): return R.success(page=page_summary, data=list(page_data)) + @extend_schema( + summary="漏洞验证", + tags=[_("Vulnerability")], + ) def update(self, request, pk): ser = IastVulRecheckPayloadSerializer(data=request.data) try: @@ -119,6 +136,10 @@ def update(self, request, pk): return R.success() return R.success() + @extend_schema( + summary="漏洞验证", + tags=[_("Vulnerability")], + ) def delete(self, request, pk): if IastVulRecheckPayload.objects.filter( pk=pk, user_id=request.user.id).exists(): @@ -126,6 +147,10 @@ def delete(self, request, pk): return R.success() return R.failure() + @extend_schema( + summary="漏洞验证状态", + tags=[_("Vulnerability")], + ) def status_change(self, request): mode = request.data.get('mode', 1) q = ~Q(status=-1) & Q(user_id=request.user.id) From 7e661fec473b5b32db0b6a3ce7b9462f2fcc4db7 Mon Sep 17 00:00:00 2001 From: st1020 Date: Wed, 5 Jul 2023 10:41:58 +0800 Subject: [PATCH 046/161] feat: remove some api --- dongtai_web/urls.py | 9 -- dongtai_web/views/log_api_list.py | 46 -------- dongtai_web/views/log_export_v2.py | 94 ---------------- dongtai_web/views/log_ip_list.py | 46 -------- dongtai_web/views/logs_v2.py | 172 ----------------------------- 5 files changed, 367 deletions(-) delete mode 100644 dongtai_web/views/log_api_list.py delete mode 100644 dongtai_web/views/log_export_v2.py delete mode 100644 dongtai_web/views/log_ip_list.py delete mode 100644 dongtai_web/views/logs_v2.py diff --git a/dongtai_web/urls.py b/dongtai_web/urls.py index d016d7376..ba3be9eda 100644 --- a/dongtai_web/urls.py +++ b/dongtai_web/urls.py @@ -67,10 +67,6 @@ from dongtai_web.views.project_version_list import ProjectVersionList from dongtai_web.views.project_version_update import ProjectVersionUpdate from dongtai_web.views.projects import Projects -from dongtai_web.views.logs_v2 import LogsV2Endpoint -from dongtai_web.views.log_export_v2 import LogExportV2 -from dongtai_web.views.log_ip_list import LogsIPList -from dongtai_web.views.log_api_list import LogsApiList from dongtai_web.views.sca_details import ScaDetailView @@ -476,11 +472,6 @@ path('api/v2/app_vul_summary', GetAppVulsSummary.as_view()), path('api/v2/api_route/search', NewApiRouteSearch.as_view()), path('api/v2/project_version', NewProjectVersionList.as_view()), - # 日志 - path('api/v2/logs', LogsV2Endpoint.as_view()), - path('api/v2/logs/export', LogExportV2.as_view()), - path('api/v2/logs/ip_list', LogsIPList.as_view()), - path('api/v2/logs/api_list', LogsApiList.as_view()), ]) # urlpatterns.extend(scaupload_urls) departured diff --git a/dongtai_web/views/log_api_list.py b/dongtai_web/views/log_api_list.py deleted file mode 100644 index f73702c07..000000000 --- a/dongtai_web/views/log_api_list.py +++ /dev/null @@ -1,46 +0,0 @@ -import logging - -from django.http import JsonResponse -from django.utils.translation import gettext_lazy as _ -from drf_spectacular.utils import extend_schema -from rest_framework.request import Request - -from dongtai_common.endpoint import R, UserEndPoint -from dongtai_common.models.log import IastLog -from dongtai_common.models.user import User - -logger = logging.getLogger("dongtai-webapi") - - -class LogsApiList(UserEndPoint): - name = "api-v2-logs-api-list" - description = _("Log list") - - @extend_schema( - description="Get list of logs api.", - summary="Log API List", - tags=["Logs"], - ) - def get(self, request: Request) -> JsonResponse: - user: User = request.user # type: ignore - - try: - if user.is_system_admin(): - queryset = IastLog.objects.all() - elif user.is_talent_admin(): - users = self.get_auth_users(user) - user_ids = list(users.values_list("id", flat=True)) - queryset = IastLog.objects.filter(user_id__in=user_ids) - else: - queryset = IastLog.objects.filter(user=user) - - data = list( - queryset.values_list("url", flat=True) - .distinct() - .order_by("url") - ) - - return R.success(data=data) - except Exception as e: - logger.error(e, exc_info=True) - return R.success(data=list(), msg=_("failure")) diff --git a/dongtai_web/views/log_export_v2.py b/dongtai_web/views/log_export_v2.py deleted file mode 100644 index 6c903b4a4..000000000 --- a/dongtai_web/views/log_export_v2.py +++ /dev/null @@ -1,94 +0,0 @@ -from django.http import HttpResponse -from django.utils.encoding import escape_uri_path -from django.utils.translation import gettext_lazy as _ -from drf_spectacular.utils import extend_schema -from import_export import resources -from rest_framework import serializers -from rest_framework.request import Request - -from dongtai_common.endpoint import R, UserEndPoint -from dongtai_common.models.log import IastLog -from dongtai_common.models.user import User - - -class _LogsDeleteSerializer(serializers.Serializer): - ids = serializers.Field(default=None, help_text=_("Log ids to delete")) - - -class LogResurce(resources.ModelResource): - def get_export_headers(self): - return [ - "时间", - "URL", - "Raw URL", - "模块名称", - "功能名称", - "操作类型", - "用户", - "访问IP地址", - ] - - class Meta: - model = IastLog - fields = ( - "action_time", - "url", - "raw_url", - "module_name", - "function_name", - "operate_type", - "user", - "access_ip", - ) - - -class LogExportV2(UserEndPoint): - resource_class = LogResurce - - @staticmethod - def attachment_response( - export_data, - filename: str = "download.xls", - content_type: str = "application/vnd.ms-excel", - ) -> HttpResponse: - """ - - https://segmentfault.com/q/1010000009719860 - - https://blog.csdn.net/qq_34309753/article/details/99628474 - """ - response = HttpResponse(export_data, content_type=content_type) - response["content_type"] = content_type - response["Content-Disposition"] = "attachment; filename*=utf-8''{}".format( - escape_uri_path(filename) - ) - return response - - @extend_schema( - parameters=[_LogsDeleteSerializer], - description="Export Logs.", - summary="Export Logs", - tags=["Logs"], - ) - def get(self, request: Request) -> HttpResponse: - ids: str | None = request.query_params.get("ids") - if ids: - id_list = [int(id.strip()) for id in ids.split(",")] - user: User = request.user # type: ignore - if user.is_system_admin(): - queryset = IastLog.objects.filter(id__in=id_list).filter() - elif user.is_talent_admin(): - auth_users = UserEndPoint.get_auth_users(user) - queryset = IastLog.objects.filter( - id__in=id_list, user__in=auth_users - ).filter() - else: - return R.failure(msg=_("no permission")) - resources = self.resource_class() - export_data = resources.export(queryset, False) - return self.attachment_response( - getattr(export_data, "xls"), filename="用户操作日志.xls" - ) - else: - return R.failure( - status=202, - msg=_("Export failed, error message: Log id should not be empty"), - ) diff --git a/dongtai_web/views/log_ip_list.py b/dongtai_web/views/log_ip_list.py deleted file mode 100644 index fd2aef48e..000000000 --- a/dongtai_web/views/log_ip_list.py +++ /dev/null @@ -1,46 +0,0 @@ -import logging - -from django.http import JsonResponse -from django.utils.translation import gettext_lazy as _ -from drf_spectacular.utils import extend_schema -from rest_framework.request import Request - -from dongtai_common.endpoint import R, UserEndPoint -from dongtai_common.models.log import IastLog -from dongtai_common.models.user import User - -logger = logging.getLogger("dongtai-webapi") - - -class LogsIPList(UserEndPoint): - name = "api-v2-logs-ip-list" - description = _("Log list") - - @extend_schema( - description="Get list of logs ip.", - summary="Log IP List", - tags=["Logs"], - ) - def get(self, request: Request) -> JsonResponse: - user: User = request.user # type: ignore - - try: - if user.is_system_admin(): - queryset = IastLog.objects.all() - elif user.is_talent_admin(): - users = self.get_auth_users(user) - user_ids = list(users.values_list("id", flat=True)) - queryset = IastLog.objects.filter(user_id__in=user_ids) - else: - queryset = IastLog.objects.filter(user=user) - - data = list( - queryset.values_list("access_ip", flat=True) - .distinct() - .order_by("access_ip") - ) - - return R.success(data=data) - except Exception as e: - logger.error(e, exc_info=True) - return R.success(data=list(), msg=_("failure")) diff --git a/dongtai_web/views/logs_v2.py b/dongtai_web/views/logs_v2.py deleted file mode 100644 index a6995b30c..000000000 --- a/dongtai_web/views/logs_v2.py +++ /dev/null @@ -1,172 +0,0 @@ -import logging -from datetime import datetime - -from django.core.cache import cache -from django.http import JsonResponse -from django.utils.translation import gettext_lazy as _ -from drf_spectacular.utils import extend_schema -from rest_framework import serializers -from rest_framework.request import Request -from rest_framework.serializers import ValidationError - -from dongtai_common.endpoint import R, UserEndPoint -from dongtai_common.models.log import IastLog -from dongtai_common.models.user import User - -logger = logging.getLogger("dongtai-webapi") - - -class _LogsArgsSerializer(serializers.Serializer): - pageSize = serializers.IntegerField(default=20, help_text=_("Number per page")) - page = serializers.IntegerField(min_value=1, default=1, help_text=_("Page index")) - startTime = serializers.DateTimeField(default=None, help_text=_("The start time.")) - endTime = serializers.DateTimeField(default=None, help_text=_("The end time.")) - ip = serializers.CharField(default=None, help_text=_("IP Address.")) - api = serializers.CharField(default=None, help_text=_("API URL.")) - - -class _LogsDeleteSerializer(serializers.Serializer): - ids = serializers.Field(default=None, help_text=_("Log ids to delete")) - - -class LogsV2Endpoint(UserEndPoint): - name = "api-v2-logs" - description = _("Log list") - - def make_key(self, request): - self.cache_key = f"{request.user.id}_total_logs_id" - self.cache_key_max_id = f"{request.user.id}_max_logs_id" - - def get_query_cache(self): - total = cache.get(self.cache_key) - max_id = cache.get(self.cache_key_max_id) - return total, max_id - - def set_query_cache(self, queryset): - total = queryset.values("id").count() - if total > 0: - max_id = queryset.values_list("id", flat=True).order_by("-id")[0] - else: - max_id = 0 - cache.set(self.cache_key, total, 60 * 60) - cache.set(self.cache_key_max_id, max_id, 60 * 60) - return total, max_id - - @extend_schema( - parameters=[_LogsArgsSerializer], - description="Get List of logs.", - summary="Log List", - tags=["Logs"], - ) - def get(self, request: Request) -> JsonResponse: - ser = _LogsArgsSerializer(data=request.GET) - user: User = request.user # type: ignore - try: - if ser.is_valid(True): - page: int = ser.validated_data.get("page", 1) - page_size: int = ser.validated_data.get("pageSize", 20) - start_time: datetime | None = ser.validated_data.get("startTime", None) - end_time: datetime | None = ser.validated_data.get("endTime", None) - ip: str | None = ser.validated_data.get("ip", None) - api: str | None = ser.validated_data.get("api", None) - else: - return R.failure(data="Can not validation data.") - except ValidationError as e: - return R.failure(data=e.detail) - - try: - if user.is_system_admin(): - queryset = IastLog.objects.all() - elif user.is_talent_admin(): - users = self.get_auth_users(user) - user_ids = list(users.values_list("id", flat=True)) - queryset = IastLog.objects.filter(user_id__in=user_ids) - else: - queryset = IastLog.objects.filter(user=user) - - if start_time is not None and end_time is not None: - queryset = queryset.filter(action_time__range=(start_time, end_time)) - elif start_time is not None: - queryset = queryset.filter(action_time__gte=start_time) - elif end_time is not None: - queryset = queryset.filter(action_time__lte=end_time) - - if ip is not None: - queryset = queryset.filter(access_ip=ip) - - if api is not None: - queryset = queryset.filter(url=api) - - # set cache key - self.make_key(request) - if page == 1: - total, max_id = self.set_query_cache(queryset) - else: - total, max_id = self.get_query_cache() - if not total or not max_id: - total, max_id = self.set_query_cache(queryset) - # only read log_id - cur_data = ( - queryset.filter(id__lte=max_id) - .values_list("id", flat=True) - .order_by("-id")[(page - 1) * page_size : page * page_size] - ) - cur_ids = [] - for item in cur_data: - cur_ids.append(item) - # read log detail - page_data = ( - IastLog.objects.filter(id__in=cur_ids) - .order_by("-id") - .select_related("user") - ) - data = [] - for item in page_data: - data.append( - { - "log_id": item.id, - "user_id": item.user.id, - "username": item.user.username, - "action_time": item.action_time.strftime("%Y-%m-%d %H:%M:%S"), - "url": item.url, - "raw_url": item.raw_url, - "module_name": item.module_name, - "function_name": item.function_name, - "operate_type": item.operate_type, - "access_ip": item.access_ip, - } - ) - return R.success(data=data, total=total) - except Exception as e: - logger.error(e, exc_info=True) - return R.success(data=list(), msg=_("failure")) - - @extend_schema( - parameters=[_LogsDeleteSerializer], - description="Delete Logs. " - "Query param ids is optional, if not set ids, will delete all logs", - summary="Delete Logs", - tags=["Logs"], - ) - def delete(self, request: Request) -> JsonResponse: - ids: str | None = request.query_params.get("ids", None) - user: User = request.user # type: ignore - if ids: - id_list = [int(id.strip()) for id in ids.split(",")] - if user.is_superuser == 1: - IastLog.objects.filter(id__in=id_list).delete() - return R.success(msg=_("success")) - users = self.get_auth_users(user) - user_ids = list(users.values_list("id", flat=True)) - IastLog.objects.filter(id__in=id_list, user_id__in=user_ids).delete() - return R.success(msg=_("success")) - else: - now = datetime.now() - if user.is_system_admin(): - IastLog.objects.filter(action_time__lt=now).delete() - elif user.is_talent_admin(): - users = self.get_auth_users(user) - IastLog.objects.filter(action_time__lt=now, user__in=users).delete() - else: - return R.failure(status=203, msg=_("no permission")) - return R.success() From 9790b51dc64391189142c174725f261d559443b7 Mon Sep 17 00:00:00 2001 From: Bidaya0 Date: Wed, 5 Jul 2023 10:50:09 +0800 Subject: [PATCH 047/161] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index fcfa9323f..f55f57f29 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,6 @@ The usage scenarios of "DongTai IAST" include but not limited to: ### 1. SaaS Version -- Fill out the [Online Form](https://jinshuju.net/f/I9PNmf) to register an account. - Log in to the [DongTai IAST] (). - Have a quick start with [Online Guideline](https://docs.dongtai.io/docs/category/%E5%BF%AB%E9%80%9F%E5%BC%80%E5%A7%8B). From 5a4830d0db102701b889290fec17e1a593cd257f Mon Sep 17 00:00:00 2001 From: Bidaya0 Date: Wed, 5 Jul 2023 10:50:49 +0800 Subject: [PATCH 048/161] Update README-zh.md --- README-zh.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README-zh.md b/README-zh.md index a0c462749..106993daa 100644 --- a/README-zh.md +++ b/README-zh.md @@ -58,7 +58,6 @@ ### 1. SaaS版本 -- 填写[在线问卷](https://jinshuju.net/f/I9PNmf)注册账号 - 登录[洞态IAST](https://iast.io)系统 - 根据[在线文档](https://doc.dongtai.io/docs/category/%E5%BF%AB%E9%80%9F%E5%BC%80%E5%A7%8B/)进行快速体验 From 95a7c4c48ba36b8a44da873414999d1efc825ce6 Mon Sep 17 00:00:00 2001 From: bidaya0 Date: Wed, 5 Jul 2023 11:04:29 +0800 Subject: [PATCH 049/161] deps: add shortuuid as deps . --- Pipfile | 1 + Pipfile.lock | 479 +++++++++++++++++++++++++++-------------------- requirements.txt | 23 ++- 3 files changed, 288 insertions(+), 215 deletions(-) diff --git a/Pipfile b/Pipfile index c52d3dd5f..07945d21a 100644 --- a/Pipfile +++ b/Pipfile @@ -84,6 +84,7 @@ django-seriously = "*" dataclasses-json = "*" django-silk = "*" types-python-dateutil = "*" +shortuuid = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index 547aa942d..904ceb77c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "1fb0ca598d338a01eb59a3af574501d4102c7f019117a5d6bacae7b86b5fe70c" + "sha256": "3ee3a2c2024fcd3b31ff46878e1d417463673a40bddd822b682fecb9936d4e20" }, "pipfile-spec": 6, "requires": { @@ -43,6 +43,14 @@ "markers": "python_version >= '3.6'", "version": "==5.1.1" }, + "annotated-types": { + "hashes": [ + "sha256:47cdc3490d9ac1506ce92c7aaa76c579dc3509ff11e098fc867e5130ab7be802", + "sha256:58da39888f92c276ad970249761ebea80ba544b77acddaa1a4d6cf78287d45fd" + ], + "markers": "python_version >= '3.7'", + "version": "==0.5.0" + }, "asgiref": { "hashes": [ "sha256:89b2ef2247e3b562a16eef663bc0e2e703ec6468e2fa8a5cd61cd449786d4f6e", @@ -64,7 +72,7 @@ "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04", "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015" ], - "markers": "python_full_version >= '3.7.0'", + "markers": "python_version >= '3.7'", "version": "==23.1.0" }, "autopep8": { @@ -80,7 +88,7 @@ "sha256:0f50d6be051c6b2b75bfbc8bfd85af195c5739c281d3f5b86a5640c65563614a", "sha256:1ad2eeae8e28053d729ba3373d34d9d6e210f6e4d8bf0a9c64f92bd053f1edf5" ], - "markers": "python_full_version >= '3.7.0'", + "markers": "python_version >= '3.7'", "version": "==4.1.0" }, "boto3": { @@ -93,11 +101,11 @@ }, "boto3-stubs": { "hashes": [ - "sha256:1062612f63f154f47a4f5b7b40c2cb15debe5f44774587110da76fd292f528f0", - "sha256:bc0cc5067f55b2da628db8a73119ecccd74b27cf424af83e56526cd90beaf9f8" + "sha256:451749fc2bb0af5718bf1410473ec2e7f915bb860614cd0f6aca00c254ccf7e3", + "sha256:55b094ebbefecb0b8015451707aafeb81c2313e216dbbd5e2f2efff70a02db63" ], "index": "pypi", - "version": "==1.26.144" + "version": "==1.27.0" }, "botocore": { "hashes": [ @@ -109,11 +117,11 @@ }, "botocore-stubs": { "hashes": [ - "sha256:b9db32981b4deefb01784d9b196afeaca7df6f6f185d8ba7f96c02b1c3bc0d90", - "sha256:d456543af79fbdd23df76a2d7a7525cd672b4bb5b057d7e060bc117d9af71694" + "sha256:020de306ca1e18263e5a73b9142ec9901080f36d7c302ca53850483955e894ad", + "sha256:e9b23f54137bffbe7dcc08d9ca072172368cf92723aee34ec1de6e665f767c60" ], "index": "pypi", - "version": "==1.29.144" + "version": "==1.29.165" }, "celery": { "hashes": [ @@ -303,7 +311,7 @@ "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e", "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48" ], - "markers": "python_full_version >= '3.7.0'", + "markers": "python_version >= '3.7'", "version": "==8.1.3" }, "click-didyoumean": { @@ -311,7 +319,7 @@ "sha256:a0713dc7a1de3f06bc0df5a9567ad19ead2d3d5689b434768a6145bff77c0667", "sha256:f184f0d851d96b6d29297354ed981b7dd71df7ff500d82fa6d11f0856bee8035" ], - "markers": "python_version < '4' and python_full_version >= '3.6.2'", + "markers": "python_full_version >= '3.6.2' and python_full_version < '4.0.0'", "version": "==0.3.0" }, "click-plugins": { @@ -323,10 +331,11 @@ }, "click-repl": { "hashes": [ - "sha256:94b3fbbc9406a236f176e0506524b2937e4b23b6f4c0c0b2a0a83f8a64e9194b", - "sha256:cd12f68d745bf6151210790540b4cb064c7b13e571bc64b6957d98d120dacfd8" + "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9", + "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812" ], - "version": "==0.2.0" + "markers": "python_version >= '3.6'", + "version": "==0.3.0" }, "crcmod": { "hashes": [ @@ -364,11 +373,11 @@ }, "dataclasses-json": { "hashes": [ - "sha256:bc285b5f892094c3a53d558858a88553dd6a61a11ab1a8128a0e554385dcc5dd", - "sha256:c2c11bc8214fbf709ffc369d11446ff6945254a7f09128154a7620613d8fda90" + "sha256:1280542631df1c375b7bc92e5b86d39e06c44760d7e3571a537b3b8acabf2f0c", + "sha256:e9ac87b73edc0141aafbce02b44e93553c3123ad574958f0fe52a534b6707e8e" ], "index": "pypi", - "version": "==0.5.7" + "version": "==0.5.9" }, "ddt": { "hashes": [ @@ -391,16 +400,16 @@ "sha256:953019cdb9c9d2c9e47b5b12bcff3cf4746fc4598eb406076fa1fc27e6a1f15c", "sha256:dce43505fb7b1b317de7195579388df0746d90db07015ed47a85e5e44930ef93" ], - "markers": "python_full_version >= '3.7.0'", + "markers": "python_version >= '3.7'", "version": "==20230430" }, "django": { "hashes": [ - "sha256:031365bae96814da19c10706218c44dff3b654cc4de20a98bd2d29b9bde469f0", - "sha256:21cc991466245d659ab79cb01204f9515690f8dae00e5eabde307f14d24d4d7d" + "sha256:a477ab326ae7d8807dc25c186b951ab8c7648a3a23f9497763c37307a2b5ef87", + "sha256:dec2a116787b8e14962014bf78e120bba454135108e1af9e9b91ade7b2964c40" ], "index": "pypi", - "version": "==3.2.19" + "version": "==3.2.20" }, "django-celery-beat": { "hashes": [ @@ -412,11 +421,11 @@ }, "django-cors-headers": { "hashes": [ - "sha256:a971cd4c75b29974068cc36b5c595698822f1e0edd5f1b32ea42ea37326ad4aa", - "sha256:e3cbd247a1a835da4cf71a70d4214378813ea7e08337778b82cb2c1bc19d28d6" + "sha256:36a8d7a6dee6a85f872fe5916cc878a36d0812043866355438dfeda0b20b6b78", + "sha256:88a4bfae24b6404dd0e0640203cb27704a2a57fd546a429e5d821dfa53dd1acf" ], "index": "pypi", - "version": "==4.0.0" + "version": "==4.1.0" }, "django-cprofile-middleware": { "hashes": [ @@ -503,11 +512,11 @@ }, "django-simple-captcha": { "hashes": [ - "sha256:9649e66dab4e71efacbfef02f48b83b91684898352a1ab56f1686ce71033b328", - "sha256:f9a07e5e9de264ba4039c9eaad66bc48188a21ceda5fcdc2fa13c5512141c2c9" + "sha256:567ad84fa64c86508c679b8425cc1410c44b3cd6467e54f8d31cf077d9366407", + "sha256:6e1fcc4f4005f7d69ee7a2e59a7e863b5d3918f36a85a4d811498984aecc48ce" ], "index": "pypi", - "version": "==0.5.17" + "version": "==0.5.18" }, "django-stubs": { "extras": [ @@ -522,11 +531,11 @@ }, "django-stubs-ext": { "hashes": [ - "sha256:2696d6f7d8538341b060cffa9565c72ea797e866687e040b86d29cad8799e5fe", - "sha256:4b6b63e49f4ba30d93ec46f87507648c99c9de6911e651ad69db7084fd5b2f4e" + "sha256:c69d1cc46f1c4c3b7894b685a5022c29b2a36c7cfb52e23762eaf357ebfc2c98", + "sha256:fdacc65a14d2d4b97334b58ff178a5853ec8c8c76cec406e417916ad67536ce4" ], "markers": "python_version >= '3.8'", - "version": "==4.2.1" + "version": "==4.2.2" }, "django-timezone-field": { "hashes": [ @@ -553,11 +562,12 @@ }, "django-xff": { "hashes": [ - "sha256:46af595516c21966ce7d4233fe91f771c16285625566b49ac34e90737107111c", - "sha256:c352f06acc8e5be9c9ffb109635ddfcc952157aee86acf9064d2457c5f818803" + "sha256:8935dce34383d65b545564a3cc17dc8010c8cdec231b0932e6073ddc766d679d", + "sha256:d2a34e947e730d0c899b6386b28cf675edafa9d64336cdb8c9656b0a58ed80b6", + "sha256:fc87333f64d2ff2ce4ab6449e814240f9f195c290d2c8789ca81a1c98f5522ec" ], "index": "pypi", - "version": "==1.3.0" + "version": "==1.4.0" }, "djangorestframework": { "hashes": [ @@ -801,7 +811,7 @@ "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852", "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61" ], - "markers": "python_full_version >= '3.7.0'", + "markers": "python_version >= '3.7'", "version": "==3.1.2" }, "jmespath": { @@ -891,11 +901,11 @@ }, "kombu": { "hashes": [ - "sha256:10f17896f9d5dc28b80a882badbd333931c77c82aaa45cd4b84fcdc74398ff2b", - "sha256:8e10da0d5c5823dbfc0c30b5b5edb2f3b0e64c2a8378eb19d0bc0425b9c88f6f" + "sha256:48ee589e8833126fd01ceaa08f8a2041334e9f5894e5763c8486a550454551e9", + "sha256:fbd7572d92c0bf71c112a6b45163153dea5a7b6a701ec16b568c27d0fd2370f2" ], "markers": "python_version >= '3.8'", - "version": "==5.3.0rc2" + "version": "==5.3.1" }, "lxml": { "hashes": [ @@ -1050,66 +1060,66 @@ }, "markupsafe": { "hashes": [ - "sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed", - "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc", - "sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2", - "sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460", - "sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7", - "sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0", - "sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1", - "sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa", - "sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03", - "sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323", - "sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65", - "sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013", - "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036", - "sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f", - "sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4", - "sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419", - "sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2", - "sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619", - "sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a", - "sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a", - "sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd", - "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7", - "sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666", - "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65", - "sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859", - "sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625", - "sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff", - "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156", - "sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd", - "sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba", - "sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f", - "sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1", - "sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094", - "sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a", - "sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513", - "sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed", - "sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d", - "sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3", - "sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147", - "sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c", - "sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603", - "sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601", - "sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a", - "sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1", - "sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d", - "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3", - "sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54", - "sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2", - "sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6", - "sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58" - ], - "markers": "python_full_version >= '3.7.0'", - "version": "==2.1.2" + "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e", + "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e", + "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431", + "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686", + "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559", + "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc", + "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c", + "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0", + "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4", + "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9", + "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575", + "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba", + "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d", + "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3", + "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00", + "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155", + "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac", + "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52", + "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f", + "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8", + "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b", + "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24", + "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea", + "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198", + "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0", + "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee", + "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be", + "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2", + "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707", + "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6", + "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58", + "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779", + "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636", + "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c", + "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad", + "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee", + "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc", + "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2", + "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48", + "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7", + "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e", + "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b", + "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa", + "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5", + "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e", + "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb", + "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9", + "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57", + "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc", + "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2" + ], + "markers": "python_version >= '3.7'", + "version": "==2.1.3" }, "marshmallow": { "hashes": [ "sha256:90032c0fd650ce94b6ec6dc8dfeb0e3ff50c144586462c389b81a07205bedb78", "sha256:93f0958568da045b0021ec6aeb7ac37c81bfcccbb9a0e7ed8559885070b3a19b" ], - "markers": "python_full_version >= '3.7.0'", + "markers": "python_version >= '3.7'", "version": "==3.19.0" }, "marshmallow-enum": { @@ -1129,11 +1139,11 @@ }, "model-bakery": { "hashes": [ - "sha256:d38a72502979be20c9c5c2b1682faefde9da4eae935c028cc35832d484ea1039", - "sha256:d516d93315f66fc7f2cef0a85e1ca9e1ebb656cca5e383f17127f023b4bbdb8c" + "sha256:70e991dd503e3f1956632c621946bb8e71012172db740bab75cd9d2a447283a3", + "sha256:bfa5d118f91f08ef4694d4aaab168c98efc91dab086aee587e71d66a7001701a" ], - "markers": "python_full_version >= '3.7.0'", - "version": "==1.11.0" + "markers": "python_version >= '3'", + "version": "==1.12.0" }, "mypy": { "hashes": [ @@ -1177,50 +1187,47 @@ }, "mysqlclient": { "hashes": [ - "sha256:0d1cd3a5a4d28c222fa199002810e8146cffd821410b67851af4cc80aeccd97c", - "sha256:828757e419fb11dd6c5ed2576ec92c3efaa93a0f7c39e263586d1ee779c3d782", - "sha256:996924f3483fd36a34a5812210c69e71dea5a3d5978d01199b78b7f6d485c855", - "sha256:b355c8b5a7d58f2e909acdbb050858390ee1b0e13672ae759e5e784110022994", - "sha256:c1ed71bd6244993b526113cca3df66428609f90e4652f37eb51c33496d478b37", - "sha256:c812b67e90082a840efb82a8978369e6e69fc62ce1bda4ca8f3084a9d862308b", - "sha256:dea88c8d3f5a5d9293dfe7f087c16dd350ceb175f2f6631c9cf4caf3e19b7a96" + "sha256:004fe1d30d2c2ff8072f8ea513bcec235fd9b896f70dad369461d0ad7e570e98", + "sha256:04368445f9c487d8abb7a878e3d23e923e6072c04a6c320f9e0dc8a82efba14e", + "sha256:530ece9995a36cadb6211b9787f0c9e05cdab6702549bdb4236af5e9b535ed6a", + "sha256:5670679ff1be1cc3fef0fa81bf39f0cd70605ba121141050f02743eb878ac114", + "sha256:68837b6bb23170acffb43ae411e47533a560b6360c06dac39aa55700972c93b2", + "sha256:955dba905a7443ce4788c63fdb9f8d688316260cf60b20ff51ac3b1c77616ede", + "sha256:9c6b142836c7dba4f723bf9c93cc46b6e5081d65b2af807f400dda9eb85a16d0" ], "index": "pypi", - "version": "==2.1.1" + "version": "==2.2.0" }, "numpy": { "hashes": [ - "sha256:0ec87a7084caa559c36e0a2309e4ecb1baa03b687201d0a847c8b0ed476a7187", - "sha256:1a7d6acc2e7524c9955e5c903160aa4ea083736fde7e91276b0e5d98e6332812", - "sha256:202de8f38fc4a45a3eea4b63e2f376e5f2dc64ef0fa692838e31a808520efaf7", - "sha256:210461d87fb02a84ef243cac5e814aad2b7f4be953b32cb53327bb49fd77fbb4", - "sha256:2d926b52ba1367f9acb76b0df6ed21f0b16a1ad87c6720a1121674e5cf63e2b6", - "sha256:352ee00c7f8387b44d19f4cada524586f07379c0d49270f87233983bc5087ca0", - "sha256:35400e6a8d102fd07c71ed7dcadd9eb62ee9a6e84ec159bd48c28235bbb0f8e4", - "sha256:3c1104d3c036fb81ab923f507536daedc718d0ad5a8707c6061cdfd6d184e570", - "sha256:4719d5aefb5189f50887773699eaf94e7d1e02bf36c1a9d353d9f46703758ca4", - "sha256:4749e053a29364d3452c034827102ee100986903263e89884922ef01a0a6fd2f", - "sha256:5342cf6aad47943286afa6f1609cad9b4266a05e7f2ec408e2cf7aea7ff69d80", - "sha256:56e48aec79ae238f6e4395886b5eaed058abb7231fb3361ddd7bfdf4eed54289", - "sha256:76e3f4e85fc5d4fd311f6e9b794d0c00e7002ec122be271f2019d63376f1d385", - "sha256:7776ea65423ca6a15255ba1872d82d207bd1e09f6d0894ee4a64678dd2204078", - "sha256:784c6da1a07818491b0ffd63c6bbe5a33deaa0e25a20e1b3ea20cf0e43f8046c", - "sha256:8535303847b89aa6b0f00aa1dc62867b5a32923e4d1681a35b5eef2d9591a463", - "sha256:9a7721ec204d3a237225db3e194c25268faf92e19338a35f3a224469cb6039a3", - "sha256:a1d3c026f57ceaad42f8231305d4653d5f05dc6332a730ae5c0bea3513de0950", - "sha256:ab344f1bf21f140adab8e47fdbc7c35a477dc01408791f8ba00d018dd0bc5155", - "sha256:ab5f23af8c16022663a652d3b25dcdc272ac3f83c3af4c02eb8b824e6b3ab9d7", - "sha256:ae8d0be48d1b6ed82588934aaaa179875e7dc4f3d84da18d7eae6eb3f06c242c", - "sha256:c91c4afd8abc3908e00a44b2672718905b8611503f7ff87390cc0ac3423fb096", - "sha256:d5036197ecae68d7f491fcdb4df90082b0d4960ca6599ba2659957aafced7c17", - "sha256:d6cc757de514c00b24ae8cf5c876af2a7c3df189028d68c0cb4eaa9cd5afc2bf", - "sha256:d933fabd8f6a319e8530d0de4fcc2e6a61917e0b0c271fded460032db42a0fe4", - "sha256:ea8282b9bcfe2b5e7d491d0bf7f3e2da29700cec05b49e64d6246923329f2b02", - "sha256:ecde0f8adef7dfdec993fd54b0f78183051b6580f606111a6d789cd14c61ea0c", - "sha256:f21c442fdd2805e91799fbe044a7b999b8571bb0ab0f7850d0cb9641a687092b" - ], - "markers": "python_version >= '3.8'", - "version": "==1.24.3" + "sha256:0ac6edfb35d2a99aaf102b509c8e9319c499ebd4978df4971b94419a116d0790", + "sha256:26815c6c8498dc49d81faa76d61078c4f9f0859ce7817919021b9eba72b425e3", + "sha256:4aedd08f15d3045a4e9c648f1e04daca2ab1044256959f1f95aafeeb3d794c16", + "sha256:4c69fe5f05eea336b7a740e114dec995e2f927003c30702d896892403df6dbf0", + "sha256:5177310ac2e63d6603f659fadc1e7bab33dd5a8db4e0596df34214eeab0fee3b", + "sha256:5aa48bebfb41f93043a796128854b84407d4df730d3fb6e5dc36402f5cd594c0", + "sha256:5b1b90860bf7d8a8c313b372d4f27343a54f415b20fb69dd601b7efe1029c91e", + "sha256:6c284907e37f5e04d2412950960894b143a648dea3f79290757eb878b91acbd1", + "sha256:6d183b5c58513f74225c376643234c369468e02947b47942eacbb23c1671f25d", + "sha256:7412125b4f18aeddca2ecd7219ea2d2708f697943e6f624be41aa5f8a9852cc4", + "sha256:7cd981ccc0afe49b9883f14761bb57c964df71124dcd155b0cba2b591f0d64b9", + "sha256:85cdae87d8c136fd4da4dad1e48064d700f63e923d5af6c8c782ac0df8044542", + "sha256:8aa130c3042052d656751df5e81f6d61edff3e289b5994edcf77f54118a8d9f4", + "sha256:95367ccd88c07af21b379be1725b5322362bb83679d36691f124a16357390153", + "sha256:9c7211d7920b97aeca7b3773a6783492b5b93baba39e7c36054f6e749fc7490c", + "sha256:9e3f2b96e3b63c978bc29daaa3700c028fe3f049ea3031b58aa33fe2a5809d24", + "sha256:b76aa836a952059d70a2788a2d98cb2a533ccd46222558b6970348939e55fc24", + "sha256:b792164e539d99d93e4e5e09ae10f8cbe5466de7d759fc155e075237e0c274e4", + "sha256:c0dc071017bc00abb7d7201bac06fa80333c6314477b3d10b52b58fa6a6e38f6", + "sha256:cc3fda2b36482891db1060f00f881c77f9423eead4c3579629940a3e12095fe8", + "sha256:d6b267f349a99d3908b56645eebf340cb58f01bd1e773b4eea1a905b3f0e4208", + "sha256:d76a84998c51b8b68b40448ddd02bd1081bb33abcdc28beee6cd284fe11036c6", + "sha256:e559c6afbca484072a98a51b6fa466aae785cfe89b69e8b856c3191bc8872a82", + "sha256:ecc68f11404930e9c7ecfc937aa423e1e50158317bf67ca91736a9864eae0232", + "sha256:f1accae9a28dc3cda46a91de86acf69de0d1b5f4edd44a9b0c3ceb8036dfff19" + ], + "markers": "python_version >= '3.9'", + "version": "==1.25.0" }, "odfpy": { "hashes": [ @@ -1329,11 +1336,11 @@ }, "prompt-toolkit": { "hashes": [ - "sha256:23ac5d50538a9a38c8bde05fecb47d0b403ecd0662857a86f886f798563d5b9b", - "sha256:45ea77a2f7c60418850331366c81cf6b5b9cf4c7fd34616f733c5427e6abbb1f" + "sha256:04505ade687dc26dc4284b1ad19a83be2f2afe83e7a828ace0c72f3a1df72aac", + "sha256:9dffbe1d8acf91e3de75f3b544e4842382fc06c6babe903ac9acb74dc6e08d88" ], "markers": "python_full_version >= '3.7.0'", - "version": "==3.0.38" + "version": "==3.0.39" }, "pycodestyle": { "hashes": [ @@ -1423,45 +1430,98 @@ }, "pydantic": { "hashes": [ - "sha256:052d8654cb65174d6f9490cc9b9a200083a82cf5c3c5d3985db765757eb3b375", - "sha256:0c6fafa0965b539d7aab0a673a046466d23b86e4b0e8019d25fd53f4df62c277", - "sha256:1243d28e9b05003a89d72e7915fdb26ffd1d39bdd39b00b7dbe4afae4b557f9d", - "sha256:12f7b0bf8553e310e530e9f3a2f5734c68699f42218bf3568ef49cd9b0e44df4", - "sha256:1410275520dfa70effadf4c21811d755e7ef9bb1f1d077a21958153a92c8d9ca", - "sha256:16f8c3e33af1e9bb16c7a91fc7d5fa9fe27298e9f299cff6cb744d89d573d62c", - "sha256:17aef11cc1b997f9d574b91909fed40761e13fac438d72b81f902226a69dac01", - "sha256:191ba419b605f897ede9892f6c56fb182f40a15d309ef0142212200a10af4c18", - "sha256:1952526ba40b220b912cdc43c1c32bcf4a58e3f192fa313ee665916b26befb68", - "sha256:1ced8375969673929809d7f36ad322934c35de4af3b5e5b09ec967c21f9f7887", - "sha256:2e4148e635994d57d834be1182a44bdb07dd867fa3c2d1b37002000646cc5459", - "sha256:34d327c81e68a1ecb52fe9c8d50c8a9b3e90d3c8ad991bfc8f953fb477d42fb4", - "sha256:35db5301b82e8661fa9c505c800d0990bc14e9f36f98932bb1d248c0ac5cada5", - "sha256:3e59417ba8a17265e632af99cc5f35ec309de5980c440c255ab1ca3ae96a3e0e", - "sha256:42aa0c4b5c3025483240a25b09f3c09a189481ddda2ea3a831a9d25f444e03c1", - "sha256:666bdf6066bf6dbc107b30d034615d2627e2121506c555f73f90b54a463d1f33", - "sha256:66a703d1983c675a6e0fed8953b0971c44dba48a929a2000a493c3772eb61a5a", - "sha256:6a82d6cda82258efca32b40040228ecf43a548671cb174a1e81477195ed3ed56", - "sha256:6f2e754d5566f050954727c77f094e01793bcb5725b663bf628fa6743a5a9108", - "sha256:7456eb22ed9aaa24ff3e7b4757da20d9e5ce2a81018c1b3ebd81a0b88a18f3b2", - "sha256:7b1f6cb446470b7ddf86c2e57cd119a24959af2b01e552f60705910663af09a4", - "sha256:7d5b8641c24886d764a74ec541d2fc2c7fb19f6da2a4001e6d580ba4a38f7878", - "sha256:84d80219c3f8d4cad44575e18404099c76851bc924ce5ab1c4c8bb5e2a2227d0", - "sha256:88f195f582851e8db960b4a94c3e3ad25692c1c1539e2552f3df7a9e972ef60e", - "sha256:93e6bcfccbd831894a6a434b0aeb1947f9e70b7468f274154d03d71fabb1d7c6", - "sha256:93e766b4a8226e0708ef243e843105bf124e21331694367f95f4e3b4a92bbb3f", - "sha256:ab523c31e22943713d80d8d342d23b6f6ac4b792a1e54064a8d0cf78fd64e800", - "sha256:bb14388ec45a7a0dc429e87def6396f9e73c8c77818c927b6a60706603d5f2ea", - "sha256:c0ab53b609c11dfc0c060d94335993cc2b95b2150e25583bec37a49b2d6c6c3f", - "sha256:c33b60054b2136aef8cf190cd4c52a3daa20b2263917c49adad20eaf381e823b", - "sha256:ceb6a23bf1ba4b837d0cfe378329ad3f351b5897c8d4914ce95b85fba96da5a1", - "sha256:d532bf00f381bd6bc62cabc7d1372096b75a33bc197a312b03f5838b4fb84edd", - "sha256:df7800cb1984d8f6e249351139667a8c50a379009271ee6236138a22a0c0f319", - "sha256:e82d4566fcd527eae8b244fa952d99f2ca3172b7e97add0b43e2d97ee77f81ab", - "sha256:f90c1e29f447557e9e26afb1c4dbf8768a10cc676e3781b6a577841ade126b85", - "sha256:f9613fadad06b4f3bc5db2653ce2f22e0de84a7c6c293909b48f6ed37b83c61f" + "sha256:041945a6c75f2451a343674ec7d220cb7e207884fb06aaf2c16b6d0bfaf2bc39", + "sha256:7a3e3b1d0384eaa313f0810cffa475d6849794a9ae5768989518114771cb9241" ], - "markers": "python_full_version >= '3.7.0'", - "version": "==1.10.8" + "markers": "python_version >= '3.7'", + "version": "==2.0.1" + }, + "pydantic-core": { + "hashes": [ + "sha256:038876cd2dfc1319e0256995ee74cdd90df2ce03bc6060d5eaee01cc78cf3dae", + "sha256:07f02b4a474fa89be0bb0b0c42eb605d2a9c8fe11ea7f82fb754060fd0a5ac33", + "sha256:0b435c029f00b402df3ab19c07b6d8a2e26a5abbb15117b93c457e3ed40237d7", + "sha256:0cec91249c78b5697294b01e66acb819433f4111ae640b7300dd5508a522342e", + "sha256:1005ab00b3f39b044408a357b41b66709b6eca17092d2713ee4b79d85a86457b", + "sha256:13ff737b9dbda2175bf2d59f8c8d0989b9a331a50d1eb8b7e6e0fdc264af3e93", + "sha256:15cb57ca61280eca0b8d721d3629871ab239954c4cec049acf9354405836f341", + "sha256:17334cef22055154b7faf7254cc0bf86fea34a7343225b8c6d2d0e54f3533048", + "sha256:196c90996542db0151265a1fe7b32d20f5d66fc00ec12ef6f10dd6a3be5aa05f", + "sha256:1fa900836d3995ecf34b48f4687a7908b5de85f194e534a7f3a88bfeaee7e25b", + "sha256:210ed18f2c438b282a2d5710c07dfa42b8de63647f650c742ecd18a4e02a0618", + "sha256:21dcb4f0168f3877cb487dc18362b78bea1e877bcb9c6b4af7563d5e00508cc0", + "sha256:24a46c1fd078f3dc7d075200e48b219ed0876f81753201a2d97ad09165d5383f", + "sha256:26722063f83c3c4f596adc1eadfa03249afa38e75f3516684de9b57e15d07346", + "sha256:26f948f36f679d84cb1b66be40775a09275579e9bba01178dbe9b8231dcbf691", + "sha256:27338dfc0a474645d6fe2139b30f006a381f7926e80485370361d7e882a60034", + "sha256:27b3eb357a801519dcf42f6c88a3a37e140cf29be21dd5dc152cfc9fa44c34d2", + "sha256:2c351a141124c216fe4a0119ef2fa5bc70eec710e59cdd79346475b3f78d15e9", + "sha256:2c587db8f31a1c3270991945c20c2ace289fbfa7cf2d533f67f47e95c9ead83e", + "sha256:2e02faa4a5e9bd1d7cb4b056c911826f67c4bf298979f89f07c3f2446cd0cf86", + "sha256:2f45943b592070fd744660fc8e31a010ae78a6e91f8e6431c07f6dce022eb03f", + "sha256:312263dea8116f68972c41c53c0a5b5bf9f7732e7bdc978acb847ed7c9fc8207", + "sha256:3127bd2a5764ed08529ca03f8b9e486d347fb2f604cd8333ae7e55a1693073af", + "sha256:31f95633f6a3ddc8e0b850157ac0cedb8ccacbe4349310b4be6d724860d8f5c0", + "sha256:4277e1941faa5c59fddfc49dae98dc94c16288bc9a09c7b17599c8388aeadcb5", + "sha256:433b13fa81a06589dae5198dd285c5621714d4b6d75da058ba8347f8c36cb796", + "sha256:4b73e646fda49a5b503f7484a8797a36697b28b5be3adb597460f1d3d337fb82", + "sha256:4cd3178131bb7d0d3df947587d76cf9d1ab4318fe45e8ad18dafba3b1f0cda6d", + "sha256:4d5cece19558a3490ace346d70322766e670c51ce98ab9bea3e85efba6c00424", + "sha256:4ed79de66b4b9acdd613c48befe4afcbee05f6153d793df6922ffc392f46720e", + "sha256:51defb4826a28644034915ec5f5a5d3be2d56b683891343d53dfca936c634326", + "sha256:5465264bbc535a8650a3806ae5bd07e2691428004a52c961281eadce519c60cc", + "sha256:5598f9d4e063e9a64233792dc0f8a0fab8036fb66d25cfc356649667a6542bfb", + "sha256:5e931731368ea56f1787fc408757708348639ef2aa1f01e3d483ad1574780b92", + "sha256:638b474da73e71079f39a80e4d70196853c2d2fc98c3d425ce3a3ae738e2245f", + "sha256:6ccade95f48f47c898632d8dd995704924fce0f99deb7fd4f24348792769abec", + "sha256:751e6deca13d89bc5ffc4684ac8a4ea08c6c0ac8dfe12cc5d6927f249879131d", + "sha256:75bbf0045f52696aa317b38e67ef5c80a15b7aab572956df2c6fb44f3f4c8b3e", + "sha256:7d1c453a36e69ddd4ea47a8e5426a63fdcb731d18122571fbdfda23b07ad28b1", + "sha256:7e5264ed7727ab09c410a98c47430c2ab426c2edb9a7b613ca1d785dd3506b7d", + "sha256:8136e89efab6f8399bdaf5254758db37049eeaa2f39645ce999aa5162392be28", + "sha256:84f1eb4d23a37f77b20dabffe7d5971c6c8eea78bd977fcd2007704ccb540230", + "sha256:89d271bae5b6e43936e0365b387d317bd309c5e7c5645b7608b939410fb86968", + "sha256:8e63f360661847422423410ebe755258aefad8bd67e9ac516eb1d02a90bdf788", + "sha256:8ebb72dec9eefc3eb419de764d0510bbaa08e4db2b4a997576cce338a5f93c97", + "sha256:996ffb7ae3c8cb7506a58dae52bbf13a7bbbfce6c3110a2b44c20d2587e57b9b", + "sha256:9cf009170f5f93c3dad4c4f73d827541d4bb7099cf69216c091d8cdd33867255", + "sha256:9d65b216c0e55414330e46c272896d4858a30d53310aa6e58520e2fc3d122deb", + "sha256:a10ce991b6986c91fdf100611d97f76b2950a1d2c2e72be0484565bf95b03767", + "sha256:a349f816319ac85759a19ccb0e93992fe77f8e1961a389cd15c3b5c6098bcabd", + "sha256:a5576ad07f480a21b38fff2e15d2c90ab3b18f36692065235df237711b402afd", + "sha256:aab82425d10bf0624e4a7ac902eed33adae413e827b53d82ae131a10c3130208", + "sha256:b024721a940a3311328d50f7cc3d9a7aced0f5ee1fd30c0fa7cbbc542ec3a55c", + "sha256:b67ede74b43598feb405a628c83087b3df1066a388ab060cdd5333d061ecf3f5", + "sha256:b8b622793e7b7ecb25916f30e91d49424a1f10db08aa151ff7eabd29039ae15c", + "sha256:bd9587083b48ec822960a8047249c8119e82749bdf96cecc2e1975322ccb1405", + "sha256:bf4bb512eb302acbef4774f65a9ae83edfb283055de7b18b9656b8fda0869652", + "sha256:c17fd1d0fef829b364fbbd06aad286b7a73b7b93a46f1967aff1c8f78e5a250a", + "sha256:c344dd1c345b2206515edaba0e0bf4aa2b1c456822f3ac9bc0d9f7fc971a8934", + "sha256:c44ec0439fac342f773cd848b20cf28cc376670369a6d42845d180f18f2671e3", + "sha256:c63bb44c2af1250fcf6e8447b0fda17f09d28e4677910f5bc1328881ae2c527e", + "sha256:c7a2c290d6abff5abf6566aa5ea07342e74af42f4defb1f33b3b3d9e7ff1c61f", + "sha256:c815a0908065dd8eae0740e55063fcf730c5ef86edf6210ecd53ace3a85c9911", + "sha256:c9f856a5c8938f2e0c7bb337f09d5212afd390627929c53e5f0c5944c99732fc", + "sha256:ca833856df881b9809747131c38bf7b6af7262ab2c77a2834b9e9d64cf43ab4e", + "sha256:d088fdc5cc709a715cf9f49e698a5690cc00616d3379e55d07423e628a21a097", + "sha256:d2db12d32b3b83c3d1a2044f9ba31aca9a8224c7eb15d949bdae3e826ee8c6ec", + "sha256:d4677520ade160805ad55a6418db7beea9dea34f0a091da1f0bcf09c66091b54", + "sha256:dbe2b50a4c3bcc9962449eea1c73d2e509a4e3a96df38511b898eea768fde4a4", + "sha256:dc901bb6ffe6d983903242dd7495660161b8901307c5280534fee3b0a90f98e6", + "sha256:e28d86253cdc638d084751bcc1217944370c567722d377c1364fd1433d0a41f9", + "sha256:e5bcca875379fab98c7b8b4ddfe932844d9ac7dc0a850c5afa414d17988aed93", + "sha256:e6973ccb84a532e35b6a9f7f8d6024688186d950278700d408836219aa5b6164", + "sha256:e7d6a9e510ae4ea02db709472102fa7b59d48441a6c0419a7d21d0b96672a469", + "sha256:e7d8df9e29ecc2930d27fccde99ae86c1dfc42c1f92e81715df2a7dc1f7f466e", + "sha256:f295db65d4de14c0b46168a6db73be34b8fe4e3e2699a9c574b37412d0dd2a41", + "sha256:f8119485a74487780fecf8c03cce66a2fb13da2e68f4219af7aca9d0eb8ff64d", + "sha256:fb3d452def28f86fcec749659fea183650c23aa46ae4d8a9996463a1793587b5", + "sha256:fb6551210cef7423d68eaaeab60a9445e17edd33d251b2ab6c783afce9811df8", + "sha256:fc4cb821dc67963463f8d8be6dca8933210d050009b32f683d02444a3d5f1e02", + "sha256:fede91ea67570eb296d4ae88aecb9c51a46cdccb35a388dba759183ba84c61d6" + ], + "markers": "python_version >= '3.7'", + "version": "==2.0.2" }, "pymysql": { "hashes": [ @@ -1473,11 +1533,11 @@ }, "pyparsing": { "hashes": [ - "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb", - "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc" + "sha256:d554a96d1a7d3ddaf7183104485bc19fd80543ad6ac5bdb6426719d766fb06c1", + "sha256:edb662d6fe322d6e990b1594b5feaeadf806803359e3d4d42f11e295e588f0ea" ], "markers": "python_full_version >= '3.6.8'", - "version": "==3.0.9" + "version": "==3.1.0" }, "pyre2": { "hashes": [ @@ -1643,7 +1703,7 @@ "sha256:3c0da2d074bf35d6870ef157158641178a4204a6e689e82546083e31e0311346", "sha256:640bb492711f4c0c0905e1f62b6aaeb771881935ad27884852411f8e9cacbca9" ], - "markers": "python_full_version >= '3.7.0'", + "markers": "python_version >= '3.7'", "version": "==0.6.1" }, "setuptools": { @@ -1654,6 +1714,14 @@ "index": "pypi", "version": "==65.5.1" }, + "shortuuid": { + "hashes": [ + "sha256:27ea8f28b1bd0bf8f15057a3ece57275d2059d2b0bb02854f02189962c13b6aa", + "sha256:fc75f2615914815a8e4cb1501b3a513745cb66ef0fd5fc6fb9f8c3fa3481f789" + ], + "index": "pypi", + "version": "==1.0.11" + }, "simhash": { "hashes": [ "sha256:49c5d81f31254f7e3f71dc2f5a245625c3d6143584478fdf4ffd2e63e4929366", @@ -1690,42 +1758,42 @@ "yaml" ], "hashes": [ - "sha256:0f9c45141195c472202f30d82c0c035bf7e0dd7e4da2257815e506acff4ab364", - "sha256:77ea97faf6f92a7e198c05bd0c690f3cba57b83ea45a636b72f967cb6fe6f160" + "sha256:9821caa9eca6062ff7299fa645e737aecff982e6b2b42046928a6413c8dabfd9", + "sha256:f6661dfc45e1d4f51fa8a6239f9c8349380859a5bfaa73280645f046d6c96e33" ], - "markers": "python_full_version >= '3.7.0'", - "version": "==3.4.0" + "markers": "python_version >= '3.8'", + "version": "==3.5.0" }, "tomli": { "hashes": [ "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" ], - "markers": "python_full_version >= '3.7.0'", + "markers": "python_version < '3.11'", "version": "==2.0.1" }, "types-awscrt": { "hashes": [ - "sha256:50fe7610aa40550a23d79d6167b2b8536281f038f42846eda0e520d6e3e01787", - "sha256:763d8d543f145d51cd16ea407d079608b126941d24525e16cb1de31d949ff563" + "sha256:25ed116be03203fddd889bb46db2923f655490c9e7fe4734176ebef53c0e782a", + "sha256:c7ec33cf74734f79bfcf0f9a76aaa7feac40c34f457a3de7977be1d8fdf02a37" ], - "markers": "python_version < '4' and python_full_version >= '3.7.0'", - "version": "==0.16.19" + "markers": "python_version >= '3.7' and python_version < '4.0'", + "version": "==0.16.21" }, "types-pymysql": { "hashes": [ - "sha256:672f0e3e5070da2dc58448d289b878c5c99febb37c9f2cbe309779301ae914fb", - "sha256:8ea7083a3bd37c4e77f38d8d93e488fba3bb03e3d3d41cc52cd92110375ed517" + "sha256:cbd0c123a8116f7b99970a7c663399bb3b4bb9d97b8f843909c5cc27abea064f", + "sha256:e350c8920455eb5cb3f8a65fd95a1350340e396f6f1451f0fe1d035240929969" ], "index": "pypi", - "version": "==1.0.19.7" + "version": "==1.1.0.0" }, "types-pyopenssl": { "hashes": [ - "sha256:43e307e8dfb3a7a8208a19874ca060305f460c529d4eaca8a2669ea89499f244", - "sha256:ba803a99440b0c2e9ab4e197084aeefc55bdfe8a580d367b2aa4210810a21240" + "sha256:0568553f104466f1b8e0db3360fbe6770137d02e21a1a45c209bf2b1b03d90d4", + "sha256:beeb5d22704c625a1e4b6dc756355c5b4af0b980138b702a9d9f932acf020903" ], - "version": "==23.2.0.0" + "version": "==23.2.0.1" }, "types-python-dateutil": { "hashes": [ @@ -1752,11 +1820,11 @@ }, "types-redis": { "hashes": [ - "sha256:2fe82f374d9dddf007deaf23d81fddcfd9523d9522bf11523c5c43bc5b27099e", - "sha256:bf8692252038dbe03b007ca4fde87d3ae8e10610854a6858e3bf5d01721a7c4b" + "sha256:a98f3386f44d045057696f3efc8869c53dda0060610e0fe3d8a4d391e2a8916a", + "sha256:d0efcd96f65fd2036437c29d8c12566cfdc549345d73eddacb0488b81aff9f9e" ], "index": "pypi", - "version": "==4.5.5.2" + "version": "==4.6.0.2" }, "types-requests": { "hashes": [ @@ -1771,7 +1839,7 @@ "sha256:6d1ac1dedac750d570428362acdf60fdd4f277b0788855c3894d3226756b2bfb", "sha256:75ac1d7143d58c1e6af467cfd4a96c67ee058a3adf7c249d9309999e1f5f41e4" ], - "markers": "python_version < '4' and python_full_version >= '3.7.0'", + "markers": "python_version >= '3.7' and python_version < '4.0'", "version": "==0.6.1" }, "types-urllib3": { @@ -1784,11 +1852,11 @@ }, "typing-extensions": { "hashes": [ - "sha256:06006244c70ac8ee83fa8282cb188f697b8db25bc8b4df07be1873c43897060c", - "sha256:3a8b36f13dd5fdc5d1b16fe317f5668545de77fa0b8e02006381fd49d731ab98" + "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36", + "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2" ], "index": "pypi", - "version": "==4.6.2" + "version": "==4.7.1" }, "typing-inspect": { "hashes": [ @@ -1867,10 +1935,11 @@ }, "zope.event": { "hashes": [ - "sha256:73d9e3ef750cca14816a9c322c7250b0d7c9dbc337df5d1b807ff8d3d0b9e97c", - "sha256:81d98813046fc86cc4136e3698fee628a3282f9c320db18658c21749235fce80" + "sha256:2832e95014f4db26c47a13fdaef84cef2f4df37e66b59d8f1f4a8f319a632c26", + "sha256:bac440d8d9891b4068e2b5a2c5e2c9765a9df762944bda6955f96bb9b91e67cd" ], - "version": "==4.6" + "markers": "python_version >= '3.7'", + "version": "==5.0" }, "zope.interface": { "hashes": [ @@ -1905,7 +1974,7 @@ "sha256:f72f23bab1848edb7472309e9898603141644faec9fd57a823ea6b4d1c4c8995", "sha256:fa90bac61c9dc3e1a563e5babb3fd2c0c1c80567e815442ddbe561eadc803b30" ], - "markers": "python_full_version >= '3.7.0'", + "markers": "python_version >= '3.7'", "version": "==6.0" } }, diff --git a/requirements.txt b/requirements.txt index d27cd7b87..d6e179c3a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,15 +3,16 @@ aliyun-python-sdk-core==2.13.36 aliyun-python-sdk-core-v3==2.13.33 aliyun-python-sdk-kms==2.16.1 amqp==5.1.1 ; python_version >= '3.6' +annotated-types==0.5.0 ; python_version >= '3.7' asgiref==3.7.2 async-timeout==4.0.2 ; python_version >= '3.6' attrs==23.1.0 ; python_version >= '3.7' autopep8==2.0.2 ; python_version >= '3.6' billiard==4.1.0 ; python_version >= '3.7' boto3==1.24.59 -boto3-stubs==1.26.162 +boto3-stubs==1.27.0 botocore==1.27.91 -botocore-stubs==1.29.162 +botocore-stubs==1.29.165 celery==5.3.0rc1 celery-singleton==0.3.1 certifi==2023.5.7 @@ -24,11 +25,11 @@ click-plugins==1.1.1 click-repl==0.3.0 ; python_version >= '3.6' crcmod==1.7 cryptography==41.0.0 -dataclasses-json==0.5.8 +dataclasses-json==0.5.9 ddt==1.6.0 defusedxml==0.7.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' diff-match-patch==20230430 ; python_version >= '3.7' -django==3.2.19 +django==3.2.20 django-celery-beat==2.2.0 django-cors-headers==4.1.0 django-cprofile-middleware==1.0.5 @@ -42,7 +43,7 @@ django-redis==5.2.0 django-rest-framework-proxy==1.6.0 django-seriously==0.4.0 django-silk==5.0.3 -django-simple-captcha==0.5.17 +django-simple-captcha==0.5.18 django-stubs[compatible-mypy]==1.15.0 django-stubs-ext==4.2.2 ; python_version >= '3.8' django-timezone-field==4.2.3 ; python_version >= '3.5' @@ -89,12 +90,13 @@ oss2==2.13.1 packaging==21.3 pep8==1.7.1 pillow==9.3.0 -prompt-toolkit==3.0.38 ; python_full_version >= '3.7.0' +prompt-toolkit==3.0.39 ; python_full_version >= '3.7.0' pycodestyle==2.10.0 ; python_version >= '3.6' pycparser==2.21 pycryptodome==3.18.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' pycryptodomex==3.14.1 -pydantic==1.10.9 ; python_version >= '3.7' +pydantic==2.0.1 ; python_version >= '3.7' +pydantic-core==2.0.2 ; python_version >= '3.7' pymysql==1.0.2 pyparsing==3.1.0 ; python_full_version >= '3.6.8' pyre2==0.3.6 @@ -110,22 +112,23 @@ requests==2.31.0 result==0.8.0 s3transfer==0.6.1 ; python_version >= '3.7' setuptools==65.5.1 +shortuuid==1.0.11 simhash==2.1.2 six==1.15.0 sqlparse==0.4.4 ; python_version >= '3.5' tablib[html,ods,xls,xlsx,yaml]==3.5.0 ; python_version >= '3.8' tomli==2.0.1 ; python_version < '3.11' types-awscrt==0.16.21 ; python_version >= '3.7' and python_version < '4.0' -types-pymysql==1.0.19.7 +types-pymysql==1.1.0.0 types-pyopenssl==23.2.0.1 types-python-dateutil==2.8.19.13 types-pytz==2023.3.0.0 types-pyyaml==6.0.12.10 -types-redis==4.6.0.0 +types-redis==4.6.0.2 types-requests==2.31.0.1 types-s3transfer==0.6.1 ; python_version >= '3.7' and python_version < '4.0' types-urllib3==1.26.25.13 -typing-extensions==4.6.3 +typing-extensions==4.7.1 typing-inspect==0.9.0 tzdata==2023.3 ; python_version >= '2' uritemplate==4.1.1 ; python_version >= '3.6' From b9f1099dc0f8f08f33fd0fe14ceff707304da792 Mon Sep 17 00:00:00 2001 From: st1020 Date: Wed, 5 Jul 2023 14:32:40 +0800 Subject: [PATCH 050/161] feat: resolve some simple code conflicts --- dongtai_common/engine/compatibility.py | 5 ++++- dongtai_common/models/url_blacklist.py | 29 +++++++++++++++++++++----- dongtai_common/translation.py | 2 +- dongtai_web/apitimelog/views.py | 2 +- dongtai_web/apps.py | 4 +++- static/templates/pdf.html | 6 ++++-- 6 files changed, 37 insertions(+), 11 deletions(-) diff --git a/dongtai_common/engine/compatibility.py b/dongtai_common/engine/compatibility.py index 7564b25cd..f6b70f645 100644 --- a/dongtai_common/engine/compatibility.py +++ b/dongtai_common/engine/compatibility.py @@ -62,7 +62,10 @@ def parse_target_value_length(target_value: str) -> int: if not target_value: return 0 position = target_value.rfind('*') - len_of_origin = int(target_value[position + 1::]) + try: + len_of_origin = int(target_value[position + 1::]) + except ValueError as e: + return len(target_value) return len_of_origin diff --git a/dongtai_common/models/url_blacklist.py b/dongtai_common/models/url_blacklist.py index 2aaad3030..f48f5f0c3 100644 --- a/dongtai_common/models/url_blacklist.py +++ b/dongtai_common/models/url_blacklist.py @@ -4,7 +4,11 @@ from django.db.models import IntegerChoices from django.utils.translation import gettext_lazy as _ from typing import List, Dict - +from dongtai_common.models.project import ( + IastProject, + IastProjectTemplate, + VulValidation, +) class TargetOperator(IntegerChoices): EQUAL = 1, _("等于") @@ -28,11 +32,11 @@ class TargetScope(IntegerChoices): class State(IntegerChoices): ENABLE = 1, _("ENABLE") - DISABLE = 2, _("DISABLE") + DISABLE = 0, _("DISABLE") class IastAgentBlackRule(models.Model): - user = models.ForeignKey(User, models.DO_NOTHING) + user = models.ForeignKey(User, models.DO_NOTHING, default=-1) scope = models.IntegerField( choices=TargetScope.choices, blank=True, @@ -44,6 +48,8 @@ class IastAgentBlackRule(models.Model): blank=True, null=True, ) + project = models.ForeignKey(IastProject, models.CASCADE, default=-1) + project_template = models.ForeignKey(IastProjectTemplate, models.CASCADE, default=-1) class Meta: managed = get_managed() @@ -76,9 +82,22 @@ def to_agent_rule(self) -> Dict: def create_blacklist_rule(target_type: TargetType, operator: TargetOperator, - value: str, user_id: int, state: State): + value: str, state: State, **kwargs): ruledetail = IastAgentBlackRuleDetail.objects.create( target_type=target_type, operator=operator, value=value) - rule = IastAgentBlackRule.objects.create(user_id=user_id, state=state) + rule = IastAgentBlackRule.objects.create(state=state, **kwargs) ruledetail.rule = rule ruledetail.save() + +def update_blacklist_rule(target_type: TargetType, operator: TargetOperator, + value: str, user_id: int, state: State, + rule_id: int): + ruledetail = IastAgentBlackRuleDetail.objects.create( + target_type=target_type, operator=operator, value=value) + rule = IastAgentBlackRule.objects.filter(user_id=user_id, pk=rule_id).first() + rule.state = state + rule.save() + rule.iastagentblackruledetail_set.all().delete() + ruledetail.rule = rule + ruledetail.save() + diff --git a/dongtai_common/translation.py b/dongtai_common/translation.py index c0fb92b78..f1e17d005 100644 --- a/dongtai_common/translation.py +++ b/dongtai_common/translation.py @@ -31,7 +31,7 @@ class IastVulLevelTranslationOptions(TranslationOptions): @register(IastDeployDesc) class IastDeployDescTranslationOptions(TranslationOptions): - fields = ('desc',) + fields = ('desc', ) @register(IastDocument) diff --git a/dongtai_web/apitimelog/views.py b/dongtai_web/apitimelog/views.py index c07a4f9ea..45ee8089e 100644 --- a/dongtai_web/apitimelog/views.py +++ b/dongtai_web/apitimelog/views.py @@ -1,4 +1,4 @@ -from apitimelog.middleware import REQUEST_DICT +from dongtai_web.apitimelog.middleware import REQUEST_DICT # Create your views here. from dongtai_common.endpoint import UserEndPoint diff --git a/dongtai_web/apps.py b/dongtai_web/apps.py index f9a31287b..a6c3a7747 100644 --- a/dongtai_web/apps.py +++ b/dongtai_web/apps.py @@ -18,7 +18,9 @@ def ready(self): from dongtai_engine.plugins.project_status import update_project_status if AUTO_UPDATE_HOOK_STRATEGY and not validate_hook_strategy_update(): - print("enable auto_update_hook_strategy updating hook strategy from file") + print( + "enable auto_update_hook_strategy updating hook strategy from file" + ) Command().handle() init_schema() diff --git a/static/templates/pdf.html b/static/templates/pdf.html index d6f850191..c8c751405 100644 --- a/static/templates/pdf.html +++ b/static/templates/pdf.html @@ -334,7 +334,8 @@ .main-container .bugslist .item { width: 25%; height: 100%; - display: flex; + display: -webkit-box; + display: flex; } .main-container .bugslist .item .rect { @@ -352,7 +353,8 @@ margin: 1mm 0; width: 100%; background-color: white; - display: flex; + display: -webkit-box; + display: flex; } .main-container .bugslist .tale-item > div { From fb125be7cc411ae36853dadf5598503e6df33274 Mon Sep 17 00:00:00 2001 From: st1020 Date: Wed, 5 Jul 2023 14:55:40 +0800 Subject: [PATCH 051/161] feat: add send notify signal --- dongtai_engine/plugins/strategy_headers.py | 6 ++++++ dongtai_engine/signals/__init__.py | 2 +- dongtai_engine/signals/handlers/vul_handler.py | 9 +++++++++ dongtai_engine/signals/signals.py | 1 + .../report/handler/hardencode_vul_handler.py | 6 ++++++ dongtai_protocol/report/handler/narmal_vul_handler.py | 7 +++++++ 6 files changed, 30 insertions(+), 1 deletion(-) diff --git a/dongtai_engine/plugins/strategy_headers.py b/dongtai_engine/plugins/strategy_headers.py index e73b239b8..5612f2402 100644 --- a/dongtai_engine/plugins/strategy_headers.py +++ b/dongtai_engine/plugins/strategy_headers.py @@ -20,6 +20,7 @@ from dongtai_common.models.header_vulnerablity import IastHeaderVulnerability, IastHeaderVulnerabilityDetail from django.db import IntegrityError from dongtai_engine.plugins.project_time_update import project_time_stamp_update +from dongtai_engine.signals import send_notify class FakeSocket(): @@ -198,6 +199,11 @@ def save_vul(vul_type, method_pool, position=None, data=None): ) log_vul_found(vul.agent.user_id, vul.agent.bind_project.name, vul.agent.bind_project_id, vul.id, vul.strategy.vul_name) # type: ignore + send_notify.send_robust( + sender=save_vul, + vul_id=vul.id, + department_id=method_pool.agent.department_id, + ) cache.delete(cache_key) header_vul = None if not IastHeaderVulnerability.objects.filter( diff --git a/dongtai_engine/signals/__init__.py b/dongtai_engine/signals/__init__.py index 106415e5b..7e164dcec 100644 --- a/dongtai_engine/signals/__init__.py +++ b/dongtai_engine/signals/__init__.py @@ -5,4 +5,4 @@ # project: dongtai-engine # inappropriate implementation to trigger task of load handlers -from .signals import vul_found +from .signals import vul_found, send_notify diff --git a/dongtai_engine/signals/handlers/vul_handler.py b/dongtai_engine/signals/handlers/vul_handler.py index f90f38476..17a3428a2 100644 --- a/dongtai_engine/signals/handlers/vul_handler.py +++ b/dongtai_engine/signals/handlers/vul_handler.py @@ -22,6 +22,7 @@ from collections import defaultdict from dongtai_common.models.profile import IastProfile from dongtai_engine.plugins.project_time_update import project_time_stamp_update +from dongtai_engine.signals import send_notify def equals(source, target): @@ -356,6 +357,12 @@ def save_vul(vul_meta, vul_level, strategy_id, vul_stack, top_stack, ) log_vul_found(vul.agent.user_id, vul.agent.bind_project.name, # type: ignore vul.agent.bind_project_id, vul.id, vul.strategy.vul_name) # type: ignore + send_notify.send_robust( + sender=save_vul, + vul_id=vul.id, + department_id=vul_meta.agent.department_id, + ) + cache.delete(cache_key) #delete if exists more than one departured use redis lock #IastVulnerabilityModel.objects.filter( @@ -560,3 +567,5 @@ def handler_vul(vul_meta, vul_level, strategy_id, vul_stack, top_stack, create_vul_recheck_task(vul_id=vul.id, agent=vul.agent, timestamp=timestamp) + + diff --git a/dongtai_engine/signals/signals.py b/dongtai_engine/signals/signals.py index 03f914e93..6e11e8d1a 100644 --- a/dongtai_engine/signals/signals.py +++ b/dongtai_engine/signals/signals.py @@ -4,3 +4,4 @@ from django.dispatch import Signal vul_found = Signal() +send_notify = Signal() diff --git a/dongtai_protocol/report/handler/hardencode_vul_handler.py b/dongtai_protocol/report/handler/hardencode_vul_handler.py index 78b036dde..1ff982082 100644 --- a/dongtai_protocol/report/handler/hardencode_vul_handler.py +++ b/dongtai_protocol/report/handler/hardencode_vul_handler.py @@ -22,6 +22,7 @@ from rest_framework.serializers import ValidationError from dongtai_web.vul_log.vul_log import log_vul_found from dongtai_common.models.agent import IastAgent +from dongtai_engine.signals import send_notify logger = logging.getLogger('dongtai.openapi') @@ -130,3 +131,8 @@ def save(self): log_vul_found(iast_vul.agent.user_id, iast_vul.agent.bind_project.name, iast_vul.agent.bind_project_id, iast_vul.id, # type: ignore iast_vul.strategy.vul_name) + send_notify.send_robust( + sender=self.__class__, + vul_id=iast_vul.id, + department_id=self.agent.department_id, + ) diff --git a/dongtai_protocol/report/handler/narmal_vul_handler.py b/dongtai_protocol/report/handler/narmal_vul_handler.py index ca1bb463a..6a6020518 100644 --- a/dongtai_protocol/report/handler/narmal_vul_handler.py +++ b/dongtai_protocol/report/handler/narmal_vul_handler.py @@ -22,6 +22,7 @@ from dongtai_common.models.header_vulnerablity import IastHeaderVulnerability from django.db import IntegrityError from dongtai_protocol import utils +from dongtai_engine.signals import send_notify logger = logging.getLogger('dongtai.openapi') @@ -277,6 +278,12 @@ def save(self): iast_vul.agent.bind_project.name, iast_vul.agent.bind_project_id, iast_vul.id, # type: ignore iast_vul.strategy.vul_name) + send_notify.send_robust( + sender=self.__class__, + vul_id=iast_vul.id, + department_id=self.agent.department_id, + ) + IastVulnerabilityModel.objects.filter( strategy_id=iast_vul.strategy_id, uri=iast_vul.uri, From 31972cccf2dae926ba5be019be6ae79b646c7048 Mon Sep 17 00:00:00 2001 From: st1020 Date: Wed, 5 Jul 2023 16:26:51 +0800 Subject: [PATCH 052/161] feat: move tests --- test/signals/handlers/vul_handler.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/signals/handlers/vul_handler.py b/test/signals/handlers/vul_handler.py index 31c2ceed0..69c428f52 100644 --- a/test/signals/handlers/vul_handler.py +++ b/test/signals/handlers/vul_handler.py @@ -12,8 +12,6 @@ class VulHandlerTest(DongTaiTestCase): def test_send_vul_notify(self): from dongtai_common.models.vulnerablity import IastVulnerabilityModel vul = IastVulnerabilityModel.objects.filter(id=2208).first() - from dongtai_engine.signals.handlers import send_vul_notify - send_vul_notify(vul) def test_create_notify_config(self): web_hook_config = { @@ -24,7 +22,7 @@ def test_create_notify_config(self): from dongtai_common.models.notify_config import IastNotifyConfig notify_config = IastNotifyConfig.objects.create( notify_type=IastNotifyConfig.WEB_HOOK, - notify_metadata=json.dumps(web_hook_config), + notify_meta_data=json.dumps(web_hook_config), user_id=18 ) From a4cec4068fbdc61745d39deadaf2e094aa4ce0aa Mon Sep 17 00:00:00 2001 From: st1020 Date: Wed, 5 Jul 2023 16:55:35 +0800 Subject: [PATCH 053/161] feat: move settings and tasks --- dongtai_conf/celery.py | 12 ++++++++++++ dongtai_conf/settings.py | 16 ++++++++++++---- test/core/tasks.py | 4 ---- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/dongtai_conf/celery.py b/dongtai_conf/celery.py index dbca3ecc9..abd981fe9 100644 --- a/dongtai_conf/celery.py +++ b/dongtai_conf/celery.py @@ -92,6 +92,18 @@ configs["DJANGO_CELERY_BEAT_TZ_AWARE"] = False configs["CELERY_BEAT_SCHEDULER"] = 'django_celery_beat.schedulers:DatabaseScheduler' +try: + from dongtai_conf.celery_extend import configs as extend_config + + for k, v in extend_config: + config = configs.get(k, None) + if isinstance(v, dict) and isinstance(config, dict): + config.update(v) + elif isinstance(v, list) and isinstance(config, list): + config.extend(v) +except ImportError: + pass + app.namespace = 'CELERY' app.conf.update(configs) diff --git a/dongtai_conf/settings.py b/dongtai_conf/settings.py index 2cfc7fe65..39b5aaea2 100644 --- a/dongtai_conf/settings.py +++ b/dongtai_conf/settings.py @@ -156,6 +156,13 @@ def get_installed_apps(): 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] +try: + from dongtai_conf.settings_extend import MIDDLEWARE as MIDDLEWARE_EXTEND + + MIDDLEWARE.extend(MIDDLEWARE_EXTEND) +except ImportError: + pass + XFF_TRUSTED_PROXY_DEPTH = 20 CSRF_COOKIE_NAME = "DTCsrfToken" @@ -509,11 +516,12 @@ def safe_execute(default, exception, function, *args): There are two authentication methods. You can obtain csrf_token and sessionid through the login process, or access the corresponding API through the user's corresponding Token. The Token method is recommended here, and users can find it in the Agent installation interface such as -H -'Authorization: Token {token}', here is the token corresponding to the user, the token method also requires a token like this on the request header.""" + 'Authorization: Token {token}', here is the token corresponding to the user, the token method also requires a token like this on the request header.""" ), + 'COMPONENT_SPLIT_REQUEST': + True, } -REST_FRAMEWORK[ - 'DEFAULT_SCHEMA_CLASS'] = 'drf_spectacular.openapi.AutoSchema' +REST_FRAMEWORK['DEFAULT_SCHEMA_CLASS'] = 'drf_spectacular.openapi.AutoSchema' if os.getenv('environment', None) == 'TEST' or os.getenv('CPROFILE', None) == 'TRUE': @@ -532,11 +540,11 @@ def safe_execute(default, exception, function, *args): SCA_TIMEOUT = 0 SCA_TOKEN = "" SCA_SETUP = False +DOMAIN = config.get('other', 'domain', fallback="") if os.getenv('environment', None) in ('TEST', 'PROD'): SESSION_COOKIE_DOMAIN = config.get('other', 'demo_session_cookie_domain') CSRF_COOKIE_DOMAIN = SESSION_COOKIE_DOMAIN - DOMAIN = config.get('other', 'domain') try: DOMAIN_VUL = config.get('other', 'domain_vul') diff --git a/test/core/tasks.py b/test/core/tasks.py index 41aca4386..58094c644 100644 --- a/test/core/tasks.py +++ b/test/core/tasks.py @@ -14,10 +14,6 @@ def test_vul_recheck(self): from dongtai_engine.tasks import vul_recheck vul_recheck() - def test_report(self): - from dongtai_engine.tasks import export_report - export_report() - def test_search_vul_from_replay_method_pool(self): from dongtai_engine.tasks import search_vul_from_replay_method_pool method_id = 110 From ec87e4744c8650aa227385c16967d021f75db129 Mon Sep 17 00:00:00 2001 From: st1020 Date: Wed, 5 Jul 2023 17:13:25 +0800 Subject: [PATCH 054/161] feat: resolve some simple code conflicts --- dongtai_web/dongtai_sca/models.py | 47 -------------------- dongtai_web/dongtai_sca/views/package_vul.py | 10 +---- 2 files changed, 1 insertion(+), 56 deletions(-) diff --git a/dongtai_web/dongtai_sca/models.py b/dongtai_web/dongtai_sca/models.py index 444c62012..38fb499d0 100644 --- a/dongtai_web/dongtai_sca/models.py +++ b/dongtai_web/dongtai_sca/models.py @@ -23,23 +23,6 @@ class Meta: db_table = 'sca2_package_v2' -class Vul(models.Model): - id = models.CharField(primary_key=True, max_length=50) - summary = models.CharField(max_length=255, blank=True, null=True) - details = models.TextField(blank=True, null=True) - aliases = models.JSONField(blank=True, null=True) - modified = models.DateTimeField(blank=True, null=True) - published = models.DateTimeField(blank=True, null=True) - withdrawn = models.DateTimeField(blank=True, null=True) - references = models.JSONField(null=True) - - created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True) - updated_at = models.DateTimeField(auto_now=True, blank=True, null=True) - - class Meta: - db_table = 'sca2_vul' - - class VulPackage(models.Model): cve = models.CharField(max_length=50, blank=True, null=True) ecosystem = models.CharField(max_length=255, blank=True, null=True) @@ -60,36 +43,6 @@ class Meta: db_table = 'sca2_vul_package_v2' -class VulPackageRange(models.Model): - vul_package_id = models.IntegerField(blank=True, null=True) - ecosystem = models.CharField(max_length=255, blank=True, null=True) - name = models.CharField(max_length=255, blank=True, null=True) - type = models.CharField(max_length=50, blank=True, null=True) - introduced = models.CharField(max_length=50, blank=True, null=True) - introduced_vcode = models.CharField(max_length=50, blank=True, null=True) - fixed = models.CharField(max_length=50, blank=True, null=True) - fixed_vcode = models.CharField(max_length=50, blank=True, null=True) - - created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True) - updated_at = models.DateTimeField(auto_now=True, blank=True, null=True) - - class Meta: - db_table = 'sca2_vul_package_range' - - -class VulPackageVersion(models.Model): - vul_package_id = models.IntegerField(blank=True, null=True) - ecosystem = models.CharField(max_length=255, blank=True, null=True) - name = models.CharField(max_length=255, blank=True, null=True) - version = models.CharField(max_length=255, blank=True, null=True) - - created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True) - updated_at = models.DateTimeField(auto_now=True, blank=True, null=True) - - class Meta: - db_table = 'sca2_vul_package_version' - - class VulCveRelation(models.Model): cve = models.CharField(max_length=255) cwe = models.CharField(max_length=255) diff --git a/dongtai_web/dongtai_sca/views/package_vul.py b/dongtai_web/dongtai_sca/views/package_vul.py index 2e84e15b4..8e43fa1e3 100644 --- a/dongtai_web/dongtai_sca/views/package_vul.py +++ b/dongtai_web/dongtai_sca/views/package_vul.py @@ -1,13 +1,5 @@ -from dongtai_common.models import User -from dongtai_common.models.asset import Asset from dongtai_common.models.asset_aggr import AssetAggr -from dongtai_common.models.asset_vul import IastAssetVul, IastAssetVulTypeRelation, IastVulAssetRelation -from dongtai_web.dongtai_sca.models import Package, VulPackageVersion, VulPackage, VulPackageRange, Vul, VulCveRelation, \ - PackageLicenseInfo, \ - PackageLicenseLevel -from django.http import JsonResponse -from rest_framework import views -from django.forms.models import model_to_dict +from dongtai_common.models.asset_vul import IastAssetVul, IastAssetVulTypeRelation from dongtai_common.endpoint import R, AnonymousAndUserEndPoint, UserEndPoint from django.utils.translation import gettext_lazy as _ from dongtai_web.dongtai_sca.common.sca_vul import GetScaVulData From 7c5a4c50fc26fa1fd6f91aae43c710d96f387147 Mon Sep 17 00:00:00 2001 From: st1020 Date: Wed, 5 Jul 2023 17:40:08 +0800 Subject: [PATCH 055/161] feat: remove unused view --- dongtai_web/dongtai_sca/urls.py | 4 - dongtai_web/dongtai_sca/views/package_vul.py | 223 ------------------- dongtai_web/urls.py | 13 +- 3 files changed, 6 insertions(+), 234 deletions(-) delete mode 100644 dongtai_web/dongtai_sca/views/package_vul.py diff --git a/dongtai_web/dongtai_sca/urls.py b/dongtai_web/dongtai_sca/urls.py index 0d6e755bb..0998ae4a7 100644 --- a/dongtai_web/dongtai_sca/urls.py +++ b/dongtai_web/dongtai_sca/urls.py @@ -10,17 +10,13 @@ from dongtai_web.dongtai_sca.views.newpackagevullevel import PackageVulLevels from dongtai_web.dongtai_sca.views.newpackageprojects import NewPackageRelationProject from dongtai_web.dongtai_sca.views.newpackageprojectversions import NewPackageRelationProjectVersion -from dongtai_web.dongtai_sca.views.package_vul import OnePackageVulList, AssetPackageVulList, AssetPackageVulDetail from rest_framework import routers router = routers.DefaultRouter() urlpatterns = [ path('package/', PackageList.as_view()), - path('package_vul/', OnePackageVulList.as_view()), path('asset_projects/', AssetProjects.as_view()), - path('asset_vuls/', AssetPackageVulList.as_view()), - path('asset_vul_detail/', AssetPackageVulDetail.as_view()), path('asset_ids/', AssetAggrDetailAssetIds.as_view()), ] diff --git a/dongtai_web/dongtai_sca/views/package_vul.py b/dongtai_web/dongtai_sca/views/package_vul.py deleted file mode 100644 index 8e43fa1e3..000000000 --- a/dongtai_web/dongtai_sca/views/package_vul.py +++ /dev/null @@ -1,223 +0,0 @@ -from dongtai_common.models.asset_aggr import AssetAggr -from dongtai_common.models.asset_vul import IastAssetVul, IastAssetVulTypeRelation -from dongtai_common.endpoint import R, AnonymousAndUserEndPoint, UserEndPoint -from django.utils.translation import gettext_lazy as _ -from dongtai_web.dongtai_sca.common.sca_vul import GetScaVulData -from dongtai_common.models.asset_vul import IastVulAssetRelation -from dongtai_common.models.asset import Asset -from collections import defaultdict -import logging -from drf_spectacular.utils import extend_schema - -logger = logging.getLogger('dongtai-webapi') - -LEVEL_MAP = {'critical': '严重', 'high': '高危', 'medium': '中危', 'low': '低危'} - - -class OnePackageVulList(AnonymousAndUserEndPoint): - # 查找单个漏洞下，所有的修复的高版本 - def find_fixed_versions(self, vul_package_id, ecosystem, name, version): - vul_package_ranges = VulPackageRange.objects.filter( - vul_package_id=vul_package_id, - ecosystem=ecosystem, name=name, - type__in=['ECOSYSTEM', 'SEMVER'], - # introduced__lte=version, - fixed_vcode__gte=version - ).all() - fixed_versions = [] - for vul_package_range in vul_package_ranges: - fixed_versions.append(vul_package_range.fixed) - return fixed_versions - - @extend_schema( - deprecated=True, - summary="获取一个包内的漏洞列表", - tags=["Vulnerability"], - ) - def get(self, request): - filter_fields = ['hash', 'aql', 'ecosystem', 'name', 'version'] - _filter = Package.objects.filter() - kwargs = {} - for filter_field in filter_fields: - _val = request.GET.get(filter_field, "") - if _val != "": - kwargs[filter_field] = request.GET.get(filter_field, "") - - ecosystem = request.GET.get('ecosystem', '') - name = request.GET.get('name', '') - version = request.GET.get('version', '') - - package = _filter.filter(**kwargs).first() - print(package) - if package is not None: - ecosystem = package.ecosystem - name = package.name - version = package.version - - version_code = "" - version_list = version.split('.')[0:4] - while len(version_list) != 5: - version_list.append("0") - for _version in version_list: - version_code += _version.zfill(5) - - vul_list = [] - vul_package_ids = [] - vul_package_ranges = VulPackageRange.objects.filter( - ecosystem=ecosystem, name=name, - introduced_vcode__lte=version_code, fixed_vcode__gt=version_code - ).all()[0:1000] - - for vul_package_range in vul_package_ranges: - vul_package_ids.append(vul_package_range.vul_package_id) - - vul_package_versions = VulPackageVersion.objects.filter( - ecosystem=ecosystem, name=name, version=version - ).all()[0:1000] - for vul_package_version in vul_package_versions: - vul_package_ids.append(vul_package_version.vul_package_id) - - for vul_package_id in vul_package_ids: - vul_package = VulPackage.objects.get(pk=vul_package_id) - vul = Vul.objects.get(pk=vul_package.vul_id) - vul_list.append( - { - "vul": model_to_dict(vul), - "vul_package": model_to_dict(vul_package), - "fixed_versions": self.find_fixed_versions( - vul_package_id, - ecosystem, - name, - version_code - ) - } - ) - if package is not None: - package = model_to_dict(package) - - result = { - 'data': { - "vul_list": vul_list, - "package": package, - }, - 'msg': 'success', - 'status': 201 - } - return JsonResponse(result) - - -class AssetPackageVulList(UserEndPoint): - name = "api-v1-sca-package-vuls" - description = "" - - @extend_schema( - deprecated=True, - summary="获取组件中的包的漏洞列表", - tags=[_("Vulnerability")], - ) - def get(self, request, aggr_id): - auth_users = self.get_auth_users(request.user) - departments = request.user.get_relative_department() - asset = Asset.objects.filter(pk=aggr_id, - department__in=departments).first() - if not asset: - return R.failure(msg=_('Components do not exist or no permission to access')) - # asset_aggr = AssetAggr.objects.filter( - # signature_value=asset.signature_value).first() - # if not asset_aggr: - # return R.failure(msg=_('Components do not exist or no permission to access')) - - # asset_queryset_exist = asset_queryset.filter(signature_value=asset.signature_value, - # version=asset.version, dependency_level__gt=0).exists() - # if not asset_queryset_exist: - # return R.failure(msg=_('Components do not exist or no permission to access')) - - vul_list = [] - # auth_asset_vuls = self.get_auth_asset_vuls(asset_queryset) - # asset_vuls = IastAssetVul.objects.filter(aql=asset_aggr.package_name, - # package_hash=asset_aggr.signature_value, - # package_version=asset_aggr.version).all() - auth_asset_vuls = IastAssetVul.objects.filter( - iastvulassetrelation__asset_id=aggr_id - ).select_related('level').prefetch_related( - 'iastassetvultyperelation_set__asset_vul_type', - ).all() - vul_dependency_paths = IastAssetVul.objects.filter( - iastvulassetrelation__asset_id=aggr_id - ).values( - 'iastvulassetrelation__vul_asset_metadata__vul_dependency_path', - 'pk').all() - vul_dependency_path_dict = defaultdict(str) - for vul_dependency_path in vul_dependency_paths: - if vul_dependency_path[ - 'iastvulassetrelation__vul_asset_metadata__vul_dependency_path']: - vul_dependency_path_dict[ - vul_dependency_path['pk']] = vul_dependency_path[ - 'iastvulassetrelation__vul_asset_metadata__vul_dependency_path'][ - 0] - for a_vul in auth_asset_vuls: - # vul_type_relation = IastAssetVulTypeRelation.objects.filter( - # asset_vul_id=a_vul.id) - vul_type_relation = a_vul.iastassetvultyperelation_set.all() - vul_type_str = "" - if vul_type_relation: - vul_types = [ - _i.asset_vul_type.name for _i in vul_type_relation - ] - vul_type_str = ','.join(vul_types) - try: - cve_code = a_vul.vul_cve_nums.get('cve') - except Exception as e: - logger.debug(e) - cve_code = "" - vul_list.append({ - "asset_vul_id": - a_vul.id, - "vul_title": - a_vul.vul_name, - "cve_id": - cve_code, - "sid": - a_vul.sid, - "cve_nums": - a_vul.vul_cve_nums, - "vul_type": - vul_type_str, - "level_id": - a_vul.level.id, - "level": - a_vul.level.name_value, - "origin_package": - vul_dependency_path_dict[a_vul.id], - }) - - return R.success(data=vul_list) - - -class AssetPackageVulDetail(UserEndPoint): - name = "api-v1-sca-package-vul-detail" - description = "" - - @extend_schema( - deprecated=True, - summary="获取组件中的包的漏洞详情", - tags=[_("Vulnerability")], - ) - def get(self, request, vul_id): - # 组件漏洞基础 数据读取 - asset_vul = IastAssetVul.objects.filter(id=vul_id).first() - # 用户鉴权 - departments = request.user.get_relative_department() - # 判断是否有权限 - if not asset_vul or not permission_to_read_asset_vul(departments, vul_id): - return R.failure( - msg=_('Vul do not exist or no permission to access')) - - data = GetScaVulData(asset_vul) - - return R.success(data=data) - - -def permission_to_read_asset_vul(departments, asset_vul_id: int): - return IastVulAssetRelation.objects.filter( - asset__department__in=departments, asset_vul_id=asset_vul_id).exists() diff --git a/dongtai_web/urls.py b/dongtai_web/urls.py index ba3be9eda..66d0df8cd 100644 --- a/dongtai_web/urls.py +++ b/dongtai_web/urls.py @@ -213,7 +213,7 @@ path('strategy/user/add', StrategyAdd.as_view()), path('strategy/user/list', StrategyList.as_view()), path('agent/', Agent.as_view()), - path('agent/deploy/', AgentDeploy.as_view()), + path('agent/deploy', AgentDeploy.as_view()), # path('agent/deploy/doc', AgentDeployDesc.as_view()), Departured # path('agent/deploy/info', AgentDeployInfo.as_view()), # path('agent/deploy/submit', AgentDeploySave.as_view()), @@ -427,12 +427,11 @@ DastManageEndPoint.as_view({ 'get': "get_doc_url", })), - path( - 'project/recognize_rule/', - RecognizeRuleViewSet.as_view({ - 'get': "retrieve", - 'put': "update", - })), + path('project/recognize_rule/', + RecognizeRuleViewSet.as_view({ + 'get': "retrieve", + 'put': "update", + })), path( 'project/recognize_rule', RecognizeRuleViewSet.as_view({ From 725739fa8aa27762a01b01ecc2ef31a4e2322e27 Mon Sep 17 00:00:00 2001 From: st1020 Date: Wed, 5 Jul 2023 17:56:26 +0800 Subject: [PATCH 056/161] style: fix style error --- dongtai_common/models/url_blacklist.py | 1 - dongtai_engine/signals/handlers/vul_handler.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/dongtai_common/models/url_blacklist.py b/dongtai_common/models/url_blacklist.py index f48f5f0c3..4ae82c60b 100644 --- a/dongtai_common/models/url_blacklist.py +++ b/dongtai_common/models/url_blacklist.py @@ -100,4 +100,3 @@ def update_blacklist_rule(target_type: TargetType, operator: TargetOperator, rule.iastagentblackruledetail_set.all().delete() ruledetail.rule = rule ruledetail.save() - diff --git a/dongtai_engine/signals/handlers/vul_handler.py b/dongtai_engine/signals/handlers/vul_handler.py index 17a3428a2..1b6573240 100644 --- a/dongtai_engine/signals/handlers/vul_handler.py +++ b/dongtai_engine/signals/handlers/vul_handler.py @@ -567,5 +567,3 @@ def handler_vul(vul_meta, vul_level, strategy_id, vul_stack, top_stack, create_vul_recheck_task(vul_id=vul.id, agent=vul.agent, timestamp=timestamp) - - From 933673f5a5761b147c90a04a222933451716d24e Mon Sep 17 00:00:00 2001 From: st1020 Date: Wed, 5 Jul 2023 18:42:43 +0800 Subject: [PATCH 057/161] fix: fix celery config error --- dongtai_conf/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dongtai_conf/celery.py b/dongtai_conf/celery.py index abd981fe9..a70be8587 100644 --- a/dongtai_conf/celery.py +++ b/dongtai_conf/celery.py @@ -95,7 +95,7 @@ try: from dongtai_conf.celery_extend import configs as extend_config - for k, v in extend_config: + for k, v in extend_config.items(): config = configs.get(k, None) if isinstance(v, dict) and isinstance(config, dict): config.update(v) From cafe9b02c02ffd4d980d0e556fba3a8726547a9f Mon Sep 17 00:00:00 2001 From: st1020 Date: Thu, 6 Jul 2023 11:26:45 +0800 Subject: [PATCH 058/161] feat: remove unused files --- dongtai_web/account/__init__.py | 5 ----- dongtai_web/views/profile.py | 6 ------ dongtai_web/views/tests.py | 18 ------------------ 3 files changed, 29 deletions(-) delete mode 100644 dongtai_web/account/__init__.py delete mode 100644 dongtai_web/views/tests.py diff --git a/dongtai_web/account/__init__.py b/dongtai_web/account/__init__.py deleted file mode 100644 index 2be11e1cc..000000000 --- a/dongtai_web/account/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python -# -*- coding:utf-8 -*- -# author:owefsad -# software: PyCharm -# project: lingzhi-webapi diff --git a/dongtai_web/views/profile.py b/dongtai_web/views/profile.py index 6ee6110b3..157cd93bb 100644 --- a/dongtai_web/views/profile.py +++ b/dongtai_web/views/profile.py @@ -31,9 +31,6 @@ def get(self, request, key): description=_("Modifiy Profile with key"), tags=[_('Profile')]) def post(self, request, key): - if not request.user.is_talent_admin(): - return R.failure( - msg=_("Current users have no permission to modify")) ser = ProfilepostArgsSer(data=request.data) try: if ser.is_valid(True): @@ -90,9 +87,6 @@ class ProfileBatchModifiedEndpoint(UserEndPoint): response_schema=ProfileBatchPostArgsSer, tags=[_('Profile')]) def post(self, request): - if not request.user.is_talent_admin(): - return R.failure( - msg=_("Current users have no permission to modify")) ser = ProfileBatchPostArgsSer(data=request.data, many=True) try: if ser.is_valid(True): diff --git a/dongtai_web/views/tests.py b/dongtai_web/views/tests.py deleted file mode 100644 index eb32bd1c1..000000000 --- a/dongtai_web/views/tests.py +++ /dev/null @@ -1,18 +0,0 @@ -import unittest - -from test import DongTaiTestCase -from django_mock_queries.query import MockSet, MockModel -from dongtai_web.views.api_route_search import _get_hook_type -from dongtai_common.models.vulnerablity import IastVulnerabilityModel -from unittest.mock import patch - - -#@unittest.skip("departured when logic change.") -#class TypingTestCase(DongTaiTestCase): -# hooktypes = MockSet(MockModel(pk=1, name=''), ) -# strategies = MockSet(MockModel(pk=1, vul_name='strategy'), ) -# vul = {"hook_type_id": 1, "strategy_id": 1, "level_id": 1} -# with patch('dongtai_common.models.hook_type.HookType', -# hooktypes), patch('dongtai_common.models.strategy', strategies): -# assert _get_hook_type(vul) is not None -# assert isinstance(_get_hook_type(vul), dict) From ceb45a35e5007583deed322133a6aa11a108a760 Mon Sep 17 00:00:00 2001 From: st1020 Date: Thu, 6 Jul 2023 12:22:45 +0800 Subject: [PATCH 059/161] feat: resolve code conflicts --- dongtai_conf/conf/config.ini.test | 2 +- dongtai_engine/tasks.py | 58 --- dongtai_web/dongtai_sca/common/sca_vul.py | 73 ---- dongtai_web/dongtai_sca/tasks.py | 62 --- dongtai_web/dongtai_sca/urls.py | 10 +- dongtai_web/dongtai_sca/utils.py | 413 ------------------ .../dongtai_sca/views/asset_projects.py | 241 ---------- dongtai_web/dongtai_sca/views/newpackage.py | 1 - .../dongtai_sca/views/newpackagedetail.py | 1 - .../dongtai_sca/views/newpackageprojects.py | 1 - .../views/newpackageprojectversions.py | 1 - .../dongtai_sca/views/newpackagesummary.py | 1 - .../dongtai_sca/views/newpackagevuldetail.py | 1 - .../dongtai_sca/views/newpackagevullevel.py | 1 - .../dongtai_sca/views/newpackagevuls.py | 1 - dongtai_web/dongtai_sca/views/package.py | 80 ---- dongtai_web/serializers/vul.py | 20 +- dongtai_web/urls.py | 2 - static/i18n/en/LC_MESSAGES/django.po | 4 + static/i18n/zh/LC_MESSAGES/django.po | 4 + 20 files changed, 27 insertions(+), 950 deletions(-) delete mode 100644 dongtai_web/dongtai_sca/common/sca_vul.py delete mode 100644 dongtai_web/dongtai_sca/tasks.py delete mode 100644 dongtai_web/dongtai_sca/utils.py delete mode 100644 dongtai_web/dongtai_sca/views/asset_projects.py delete mode 100644 dongtai_web/dongtai_sca/views/package.py diff --git a/dongtai_conf/conf/config.ini.test b/dongtai_conf/conf/config.ini.test index 850c10a18..01ea882a2 100644 --- a/dongtai_conf/conf/config.ini.test +++ b/dongtai_conf/conf/config.ini.test @@ -48,7 +48,7 @@ async_send_delay = 2 [log_service] host = localhost -port = 8082 +port = 8083 [other] domain = http://localhost.domain/ diff --git a/dongtai_engine/tasks.py b/dongtai_engine/tasks.py index 3f1069fe9..eacaffe2a 100644 --- a/dongtai_engine/tasks.py +++ b/dongtai_engine/tasks.py @@ -34,7 +34,6 @@ from dongtai_engine.plugins.strategy_sensitive import check_response_content from dongtai_engine.replay import Replay from dongtai_conf import settings -from dongtai_web.dongtai_sca.utils import sca_scan_asset import requests from dongtai_engine.task_base import replay_payload_data from dongtai_engine.common.queryset import get_scan_id, load_sink_strategy, get_agent @@ -297,63 +296,6 @@ def get_project_agents(agent): return agents -@shared_task(queue='dongtai-sca-task') -def update_one_sca(agent_id, package_path, package_signature, package_name, package_algorithm, package_version=''): - """ - 根据SCA数据库，更新SCA记录信息 - :return: - """ - logger.info( - f'SCA检测开始 [{agent_id} {package_path} {package_signature} {package_name} {package_algorithm} {package_version}]') - agent = IastAgent.objects.filter(id=agent_id).first() - version = package_version - if not version: - if agent.language == "JAVA": - version = package_name.split('/')[-1].replace('.jar', '').split('-')[-1] - - if version: - current_version_agents = get_project_agents(agent) - if package_signature: - asset_count = Asset.objects.values("id").filter(signature_value=package_signature, - agent__in=current_version_agents).count() - else: - package_signature = sha_1(package_name) - asset_count = Asset.objects.values("id").filter(package_name=package_name, - version=version, - agent__in=current_version_agents).count() - - if asset_count == 0: - new_level = IastVulLevel.objects.get(name="info") - asset = Asset() - asset.package_name = package_name - asset.package_path = package_path - asset.signature_value = package_signature - asset.signature_algorithm = package_algorithm - asset.version = version - asset.level_id = new_level.id - asset.vul_count = 0 - asset.language = asset.language - if agent: - asset.agent = agent - asset.project_version_id = agent.project_version_id if agent.project_version_id else 0 - asset.project_name = agent.project_name - asset.language = agent.language - asset.project_id = -1 - if agent.bind_project_id: - asset.project_id = agent.bind_project_id - asset.user_id = -1 - if agent.user_id: - asset.user_id = agent.user_id - - asset.license = '' - asset.dt = int(time.time()) - asset.save() - sca_scan_asset(asset) - else: - logger.info( - f'SCA检测开始 [{agent_id} {package_path} {package_signature} {package_name} {package_algorithm} {version}] 组件已存在') - - def sha_1(raw): sha1_str = hashlib.sha1(raw.encode("utf-8"), usedforsecurity=False).hexdigest() return sha1_str diff --git a/dongtai_web/dongtai_sca/common/sca_vul.py b/dongtai_web/dongtai_sca/common/sca_vul.py deleted file mode 100644 index 234ade419..000000000 --- a/dongtai_web/dongtai_sca/common/sca_vul.py +++ /dev/null @@ -1,73 +0,0 @@ -from dongtai_common.models.asset import Asset -from dongtai_common.models.asset_vul import IastAssetVulTypeRelation -from dongtai_web.dongtai_sca.models import VulCveRelation -from dongtai_common.models.asset_vul import IastAssetVul - - -def get_ref(refs) -> list: - if not refs: - return [] - for ref in refs: - ref['source_url'] = ref['url'] - ref['url'] = ref['content'] - return refs - -# 通过asset_vul获取 组件详情信息 -def GetScaVulData(asset_vul, asset_queryset=None): - data = {'base_info': dict(), 'poc_info': dict()} - vul_id = asset_vul.id - - data['base_info'] = {'package_name': asset_vul.aql, 'version': asset_vul.package_version, - 'safe_version': asset_vul.package_safe_version, 'language': asset_vul.package_language} - - data['base_info']['first_time'] = asset_vul.vul_publish_time - data['base_info']['last_time'] = asset_vul.vul_update_time - - data['base_info']['cwe'] = '' - data['base_info']['cnvd'] = '' - data['base_info']['cve'] = '' - data['base_info']['cnnvd'] = '' - data['base_info']['level_id'] = asset_vul.level.id - data['base_info']['level'] = asset_vul.level.name_value - vul_cve_nums = asset_vul.vul_cve_nums - if vul_cve_nums: - data['base_info']['cwe'] = vul_cve_nums['cwe'] if vul_cve_nums['cwe'] else '' - data['base_info']['cnvd'] = vul_cve_nums['cnvd'] if vul_cve_nums['cnvd'] else '' - data['base_info']['cve'] = vul_cve_nums['cve'] if vul_cve_nums['cve'] else '' - data['base_info']['cnnvd'] = vul_cve_nums['cnnvd'] if vul_cve_nums['cnnvd'] else '' - - data['base_info']['vul_title'] = asset_vul.vul_name - data['base_info']['vul_detail'] = asset_vul.vul_detail - data['base_info']['vul_cve_id'] = asset_vul.cve_code - data['base_info']['have_article'] = asset_vul.have_article - data['base_info']['have_poc'] = asset_vul.have_poc - data['base_info']['vul_type'] = '' - data['base_info']['vul_id'] = vul_id - vul_type_relation = IastAssetVulTypeRelation.objects.filter(asset_vul_id=asset_vul.id) - if vul_type_relation: - vul_types = [_i.asset_vul_type.name for _i in vul_type_relation] - data['base_info']['vul_type'] = ','.join(vul_types) - - #asset_queryset = asset_queryset.filter( - # signature_value=asset_vul.package_hash, version=asset_vul.package_version, project_id__gt=0 - #).values('project_id', 'id').all() - #if asset_queryset: - #_temp_data = {_a['project_id']: _a['id'] for _a in asset_queryset} - #asset_ids = [_temp_data[p_id] for p_id in _temp_data] - - project_list = [] - projects_data = Asset.objects.filter( - iastvulassetrelation__asset_vul_id=asset_vul.id).values( - 'project_name').distinct().all() - for project in projects_data: - project_list.append(project['project_name']) - - data['base_info']['project_names'] = project_list - - #cve_relation = VulCveRelation.objects.filter(id=asset_vul.cve_id).first() - - data['poc_info']['poc_list'] = asset_vul.poc if asset_vul.poc else [] - - references = asset_vul.references if asset_vul.references else [] - data['poc_info']['reference_link'] = get_ref(references) - return data diff --git a/dongtai_web/dongtai_sca/tasks.py b/dongtai_web/dongtai_sca/tasks.py deleted file mode 100644 index 8262dbfdd..000000000 --- a/dongtai_web/dongtai_sca/tasks.py +++ /dev/null @@ -1,62 +0,0 @@ -# !usr/bin/env python -# coding:utf-8 -# @author:zhaoyanwei -# @file: tasks.py -# @time: 2022/5/9 下午3:45 - -from dongtai_common.models import User -from dongtai_common.models.agent import IastAgent -from dongtai_common.models.asset import Asset -from celery.apps.worker import logger - -from dongtai_web.dongtai_sca.utils import sca_scan_asset - - -def refresh_all_asset_data(): - """ - todo 一次性任务，更新组件数据 - """ - logger.info('开始更新组件数据') - - iast_assets = Asset.objects.filter(dependency_level=0).all() - if iast_assets: - for asset in iast_assets: - try: - update_fields = [] - asset_agent = IastAgent.objects.filter(id=asset.agent_id).values("bind_project_id", - "project_name", - "user_id", - "project_version_id", - "language").first() - if asset_agent: - if asset_agent['bind_project_id'] != 0: - asset.project_id = asset_agent['bind_project_id'] - asset.project_name = asset_agent['project_name'] - asset.project_version_id = asset_agent['project_version_id'] - update_fields.extend(['project_id', 'project_name', 'project_version_id']) - if asset_agent['user_id'] != 0: - user = User.objects.filter(id=asset_agent['user_id']).first() - if user: - user_department = user.get_department() - user_talent = user.get_talent() - asset.department_id = user_department.id if user_department else -1 - asset.talent_id = user_talent.id if user_talent else -1 - asset.user_id = asset_agent['user_id'] - update_fields.append('user_id') - update_fields.append('talent_id') - update_fields.append('department_id') - update_fields.append('language') - asset.language = asset_agent['language'] - - asset.save(update_fields=update_fields) - - # 更新asset - sca_scan_asset(asset) - - except Exception as e: - logger.error(f'SCA组件数据更新出错，错误原因：{e}') - continue - - logger.info('组件更新数据处理完成') - - return True diff --git a/dongtai_web/dongtai_sca/urls.py b/dongtai_web/dongtai_sca/urls.py index 0998ae4a7..c14b32803 100644 --- a/dongtai_web/dongtai_sca/urls.py +++ b/dongtai_web/dongtai_sca/urls.py @@ -1,7 +1,6 @@ from django.urls import include, path -from dongtai_web.dongtai_sca.views.asset_projects import AssetProjects -from dongtai_web.dongtai_sca.views.package import PackageList, AssetAggrDetailAssetIds + from dongtai_web.dongtai_sca.views.newpackage import PackageList from dongtai_web.dongtai_sca.views.newpackagedetail import PackageDetail from dongtai_web.dongtai_sca.views.newpackagevuldetail import PackageVulDetail @@ -14,12 +13,6 @@ router = routers.DefaultRouter() -urlpatterns = [ - path('package/', PackageList.as_view()), - path('asset_projects/', AssetProjects.as_view()), - path('asset_ids/', AssetAggrDetailAssetIds.as_view()), -] - v2_urlpatterns = [ path('package/', PackageList.as_view()), path( @@ -40,6 +33,5 @@ ] urlpatterns = [ - path('sca/v1/', include(urlpatterns), name='ScaAPI'), path('api/sca/v2/', include(v2_urlpatterns), name='ScaAPI'), ] diff --git a/dongtai_web/dongtai_sca/utils.py b/dongtai_web/dongtai_sca/utils.py deleted file mode 100644 index 89b06722c..000000000 --- a/dongtai_web/dongtai_sca/utils.py +++ /dev/null @@ -1,413 +0,0 @@ -# !usr/bin/env python -# coding:utf-8 -# @author:zhaoyanwei -# @file: utils.py -# @time: 2022/5/5 下午7:26 -import json -import logging -import random -import time - -import requests -from django.conf import settings -from django.db.models import Count, Q -from django.forms import model_to_dict - -from dongtai_common.models import User -from dongtai_common.models.agent import IastAgent -from dongtai_common.models.asset import Asset -from dongtai_common.models.asset_aggr import AssetAggr -from dongtai_common.models.asset_vul import IastAssetVul, IastVulAssetRelation, IastAssetVulType, \ - IastAssetVulTypeRelation -from dongtai_common.models.vul_level import IastVulLevel -from dongtai_web.vul_log.vul_log import log_asset_vul_found -from dongtai_web.dongtai_sca.models import Package, VulPackage, VulCveRelation, PackageLicenseLevel, PackageDependency -from dongtai_common.models.asset_vul import IastAssetVulnerabilityDocument -from dongtai_conf.settings import ELASTICSEARCH_STATE - -logger = logging.getLogger(__name__) - - -def sca_scan_asset(asset): - """ - 根据SCA数据库，更新SCA记录信息 - :return: - """ - agent = asset.agent - version = asset.version - asset_package = Package.objects.filter(hash=asset.signature_value, version=version).first() - update_fields = list() - try: - logger.info('[sca_scan_asset]开始检测组件:{}/{}'.format(asset.id, asset.package_name)) - if asset_package: - package_name = asset_package.aql - version = asset_package.version - - if version: - version_code = "" - version_list = asset_package.version.split('.')[0:4] - while len(version_list) != 5: - version_list.append("0") - for _version in version_list: - version_code += _version.zfill(5) - else: - version_code = "0000000000000000000000000" - - vul_list = VulPackage.objects.filter(ecosystem=asset_package.ecosystem, name=asset_package.name, - introduced=asset_package.version).all() - - if asset_package.license: - asset.license = asset_package.license - update_fields.append('license') - # 最小修复版本-安全版本 - package_ranges = VulPackage.objects.filter(ecosystem=asset_package.ecosystem, - name=asset_package.name, - safe_vcode__gte=version_code).order_by("safe_vcode").first() - if package_ranges and asset.safe_version != package_ranges.safe_version: - asset.safe_version = package_ranges.fixed - update_fields.append('safe_version') - - # 最新版本 - last_package = Package.objects.filter(Q(ecosystem=asset_package.ecosystem) - & Q(name=asset_package.name) & ~Q(version="")).order_by( - "-version_publish_time").first() - if last_package: - asset.last_version = last_package.version - update_fields.append('last_version') - - levels_dict = dict() - vul_count = 0 - levels = [] - vul_records = [] - for vul in vul_list: - if vul.cve in vul_records: - continue - vul_records.append(vul.cve) - - _level = vul.severity - vul_cve_code = vul.cve - if vul_cve_code: - cve_relation = VulCveRelation.objects.filter( - Q(cve=vul_cve_code) | Q(cnvd=vul_cve_code) | Q(cnnvd=vul_cve_code)).first() - if cve_relation: - if _level == 'note': - _level = 'info' - if _level and _level not in levels: - levels.append(_level) - if _level not in levels_dict: - levels_dict[_level] = 1 - else: - levels_dict[_level] += 1 - vul_count += 1 - # 写入IastAssetVul - _add_vul_data(asset, asset_package, cve_relation) - - if len(levels) > 0: - if 'critical' in levels: - level = 'high' - elif 'high' in levels: - level = 'high' - elif 'medium' in levels: - level = 'medium' - elif 'low' in levels: - level = 'low' - else: - level = 'info' - else: - level = 'info' - - new_level = IastVulLevel.objects.get(name=level) - # 更新漏洞等级和各级漏洞数 - if asset.level != new_level: - asset.level = new_level - update_fields.append('level') - - if 'critical' in levels_dict: - asset.vul_critical_count = levels_dict['critical'] - update_fields.append('vul_critical_count') - else: - asset.vul_critical_count = 0 - update_fields.append('vul_critical_count') - if 'high' in levels_dict: - asset.vul_high_count = levels_dict['high'] - update_fields.append('vul_high_count') - else: - asset.vul_high_count = 0 - update_fields.append('vul_high_count') - if 'medium' in levels_dict: - asset.vul_medium_count = levels_dict['medium'] - update_fields.append('vul_medium_count') - else: - asset.vul_medium_count = 0 - update_fields.append('vul_medium_count') - - if 'low' in levels_dict: - asset.vul_low_count = levels_dict['low'] - update_fields.append('vul_low_count') - else: - asset.vul_low_count = 0 - update_fields.append('vul_low_count') - - if 'info' in levels_dict: - asset.vul_info_count = levels_dict['info'] - update_fields.append('vul_info_count') - else: - asset.vul_info_count = 0 - update_fields.append('vul_info_count') - - if asset.vul_count != vul_count: - asset.vul_count = vul_count - update_fields.append('vul_count') - - if asset.package_name != package_name: - asset.package_name = package_name - update_fields.append('package_name') - - if asset.version != version: - asset.version = version - update_fields.append('version') - - asset.dependency_level = 1 - update_fields.append('dependency_level') - - if len(update_fields) > 0: - logger.info(f'update asset {asset.id} dependency fields: {update_fields}') - asset.save(update_fields=update_fields) - else: - logger.warning('[sca_scan_asset]检测组件在组件库不存在:{}/{}'.format(asset.id, asset.package_name)) - update_asset_aggr(asset) - except Exception as e: - # import traceback - # traceback.print_exc() - logger.info("get package_vul failed:{}".format(e)) - - -# 处理IastAssetVul -def _add_vul_data(asset, asset_package, cve_relation): - try: - level_maps = dict() - _level = cve_relation.severity - vul_title = '' - vul_detail = '' - vul_have_poc = 0 - # vul_have_article = 1 if vul_info['references'] else 0 - vul_have_article = 0 - - vul_serial = '' - vul_reference = dict() - vul_type_ids = [] - # cve_relation_id = 0 - default_cwe_info = {'cwe_id': '', 'name_chinese': '未知'} - - if cve_relation: - # cve_relation_id = cve_relation.id - vul_title = cve_relation.vul_title - if cve_relation.description: - vul_detail = cve_relation.description[0]['content'] - vul_have_poc = 1 if cve_relation.poc else 0 - vul_reference = {"cve": cve_relation.cve, "cwe": cve_relation.cwe, "cnnvd": cve_relation.cnnvd, - "cnvd": cve_relation.cnvd} - vul_serial = ' | '.join([vul_reference[_i] for _i in vul_reference]) - vul_serial = vul_title + ' | ' + vul_serial - vul_cwe_id = cve_relation.cwe.split(',') - vul_type_cwe = IastAssetVulType.objects.filter(cwe_id__in=vul_cwe_id).all() - - if not vul_type_cwe: - if cve_relation.cwe_info: - for cwe in cve_relation.cwe_info: - vul_type_cwe1 = IastAssetVulType.objects.filter(cwe_id=cwe['cwe_id']).first() - if not vul_type_cwe1: - vul_type_cwe_new = IastAssetVulType.objects.create(cwe_id=cwe['cwe_id'], - name=cwe['name_chinese']) - vul_type_id = vul_type_cwe_new.id - else: - vul_type_id = vul_type_cwe1.id - vul_type_ids.append(vul_type_id) - else: - for cwe in vul_type_cwe: - vul_type_ids.append(cwe.id) - - if not vul_type_ids: - vul_type_cwe = IastAssetVulType.objects.filter(cwe_id=default_cwe_info['cwe_id']).first() - if not vul_type_cwe: - vul_type_cwe_new = IastAssetVulType.objects.create(cwe_id=default_cwe_info['cwe_id'], - name=default_cwe_info['name_chinese']) - vul_type_ids.append(vul_type_cwe_new.id) - else: - vul_type_ids.append(vul_type_cwe.id) - - vul_license = asset_package.license - vul_aql = asset_package.aql - vul_package_hash = asset_package.hash - vul_package_v = asset_package.version - vul_package_safe_version = asset.safe_version - vul_package_latest_version = asset.last_version - vul_package_language = asset.language - - license_level_info = PackageLicenseLevel.objects.filter(identifier=vul_license).first() - vul_license_level = license_level_info.level_id if license_level_info else 0 - - if _level in level_maps: - vul_level = level_maps[_level] - else: - level_obj = IastVulLevel.objects.filter(name=_level).first() - if level_obj: - level_maps[_level] = level_obj.id - vul_level = level_obj.id - else: - vul_level = 1 # critical IastVulLevel查不到归到high - asset_vul = IastAssetVul.objects.filter(cve_id=cve_relation.id, aql=vul_aql, - package_hash=vul_package_hash, - package_version=vul_package_v).first() - timestamp = int(time.time()) - if not asset_vul: - asset_vul = IastAssetVul.objects.create( - package_name=asset_package.name, - level_id=vul_level, - license=vul_license, - license_level=vul_license_level, - vul_name=vul_title, - vul_detail=vul_detail, - aql=vul_aql, - package_hash=vul_package_hash, - package_version=vul_package_v, - package_safe_version=vul_package_safe_version, - package_latest_version=vul_package_latest_version, - package_language=vul_package_language, - have_article=vul_have_article, - have_poc=vul_have_poc, - cve_id=cve_relation.id, - cve_code=cve_relation.cve, - vul_cve_nums=vul_reference, - vul_serial=vul_serial, - vul_publish_time=cve_relation.publish_time, - vul_update_time=cve_relation.update_time, - update_time=timestamp, - update_time_desc=-timestamp, - create_time=timestamp - ) - _add_asset_vul_relation(asset_vul) - if vul_type_ids: - type_relation_obj = [] - for vul_type_id in vul_type_ids: - type_relation = IastAssetVulTypeRelation(asset_vul_id=asset_vul.id, - asset_vul_type_id=vul_type_id) - type_relation_obj.append(type_relation) - - IastAssetVulTypeRelation.objects.bulk_create(type_relation_obj) - - # new vul add log - log_project_name = asset.project_name if asset.project_name else '' - log_project_id = asset.project_id if asset.project_id else 0 - log_user_id = asset.user_id if asset.user_id else 0 - log_asset_vul_found(log_user_id, log_project_name, log_project_id, asset_vul.id, asset_vul.vul_name) - else: - asset_vul.update_time = timestamp - asset_vul.update_time_desc = -timestamp - asset_vul.save() - # vul log - log_project_name = asset.project_name if asset.project_name else '' - log_project_id = asset.project_id if asset.project_id else 0 - log_user_id = asset.user_id if asset.user_id else 0 - log_asset_vul_found(log_user_id, log_project_name, log_project_id, asset_vul.id, asset_vul.vul_name) - _add_asset_vul_relation(asset_vul) - - except Exception as e: - # import traceback - # traceback.print_exc() - logger.info("_add_vul_data failed:{}".format(e)) - - -def _add_asset_vul_relation(asset_vul): - vul_assets = Asset.objects.filter(version=asset_vul.package_version, - signature_value=asset_vul.package_hash).values('id').all() - asset_vul_relations = [] - timestamp = int(time.time()) - if vul_assets: - for asset_vl in vul_assets: - relation_exist = IastVulAssetRelation.objects.filter(asset_vul_id=asset_vul.id, - asset_id=asset_vl['id']).first() - if not relation_exist: - asset_vul_relations.append(IastVulAssetRelation(asset_vul_id=asset_vul.id, asset_id=asset_vl['id'], - create_time=timestamp, status_id=1)) - - if asset_vul_relations: - IastVulAssetRelation.objects.bulk_create(asset_vul_relations) - if ELASTICSEARCH_STATE: - asset_vul_created = IastVulAssetRelation.objects.filter( - asset_vul=asset_vul, create_time=timestamp, status_id=1).all() - IastAssetVulnerabilityDocument().update(asset_vul_created) - -def update_asset_aggr(asset): - try: - project_count = 0 - asset_aggr = AssetAggr.objects.filter(signature_value=asset.signature_value, version=asset.version).first() - project_count_query = Asset.objects.filter(project_id__gt=0, signature_value=asset.signature_value, - version=asset.version).values( - 'signature_value', 'version').annotate(project_count=Count('project_id', distinct=True)) - if project_count_query: - project_count = [_['project_count'] for _ in project_count_query][0] - - if asset_aggr: - asset_aggr.version = asset.version - asset_aggr.safe_version = asset.safe_version - asset_aggr.last_version = asset.last_version - asset_aggr.level = asset.level - asset_aggr.vul_count = asset.vul_count - asset_aggr.vul_critical_count = asset.vul_critical_count - asset_aggr.vul_high_count = asset.vul_high_count - asset_aggr.vul_medium_count = asset.vul_medium_count - asset_aggr.vul_low_count = asset.vul_low_count - asset_aggr.vul_info_count = asset.vul_info_count - asset_aggr.language = asset.language - asset_aggr.license = asset.license - asset_aggr.is_del = asset.is_del - asset_aggr.project_count = project_count - asset_aggr.save() - else: - AssetAggr.objects.create( - package_name=asset.package_name, - signature_value=asset.signature_value, - version=asset.version, - safe_version=asset.safe_version, - last_version=asset.last_version, - level=asset.level, - vul_count=asset.vul_count, - vul_critical_count=asset.vul_critical_count, - vul_high_count=asset.vul_high_count, - vul_medium_count=asset.vul_medium_count, - vul_low_count=asset.vul_low_count, - project_count=project_count, - language=asset.language, - license=asset.license, - is_del=asset.is_del) - except Exception as e: - logger.error("update_asset_aggr error {}:".format(e)) - - -def get_asset_id_by_aggr_id(aggr_id, asset_ids=None): - data_ids = [] - asset_aggr = AssetAggr.objects.filter(id=aggr_id).first() - if asset_aggr: - assets = Asset.objects.filter(signature_value=asset_aggr.signature_value, version=asset_aggr.version) - if asset_ids: - assets = assets.filter(id__in=asset_ids) - asset_datas = assets.values('id').all() - for asset in asset_datas: - data_ids.append(asset['id']) - - return data_ids - - -def get_package_name_by_aql(aql): - name = '' - - if aql: - aql_split = aql.split(':') - if len(aql_split) > 0: - del aql_split[-2] - del aql_split[-1] - del aql_split[0] - name = ':'.join(aql_split) - - return name diff --git a/dongtai_web/dongtai_sca/views/asset_projects.py b/dongtai_web/dongtai_sca/views/asset_projects.py deleted file mode 100644 index 39b55977f..000000000 --- a/dongtai_web/dongtai_sca/views/asset_projects.py +++ /dev/null @@ -1,241 +0,0 @@ -# !usr/bin/env python -# coding:utf-8 -# @author:zhaoyanwei -# @file: asset_projects.py -# @time: 2022/5/7 上午7:18 -import logging - -from django.db.models import Count -from django.forms import model_to_dict -from django.utils.translation import gettext_lazy as _ - -from dongtai_common.endpoint import R, UserEndPoint -from dongtai_common.models import User -from dongtai_common.models.asset import Asset -from dongtai_common.models.asset_aggr import AssetAggr -from dongtai_common.models.asset_vul import IastAssetVul -from dongtai_common.models.project_version import IastProjectVersion -from dongtai_common.models.vul_level import IastVulLevel -from dongtai_web.serializers.sca import ScaSerializer -from dongtai_web.dongtai_sca.serializers.asset_project import AssetProjectSerializer -from typing import List, Optional, Dict -from drf_spectacular.utils import extend_schema - -logger = logging.getLogger(__name__) - - -class AssetProjects(UserEndPoint): - name = "api-v1-sca-projects" - description = "" - - @extend_schema( - deprecated=True, - summary="获取组件项目列表", - tags=["Project"], - ) - def get(self, request, aggr_id): - try: - departments = request.user.get_relative_department() - asset_queryset = Asset.objects.filter(department__in=departments, is_del=0) - asset = Asset.objects.filter(id=aggr_id).first() - #asset_aggr = AssetAggr.objects.filter( - # signature_value=asset.signature_value).first() - if not asset: - return R.failure(msg=_('Components do not exist or no permission to access')) - - asset_queryset = asset_queryset.filter( - signature_value=asset.signature_value, - version=asset.version, - project_id__gt=0).values('project_id', 'id').all() - if not asset_queryset: - return R.failure(msg=_( - 'Components do not exist or no permission to access')) - - _temp_data = {_a['project_id']: _a['id'] for _a in asset_queryset} - asset_ids = [_temp_data[p_id] for p_id in _temp_data] - - data = AssetProjectSerializer( - Asset.objects.filter(pk__in=asset_ids), many=True).data - - return R.success(data=data) - except Exception as e: - logger.error(e) - return R.failure(msg=_('Component projects query failed')) - - -class AssetVulProjects(UserEndPoint): - name = "api-v1-sca-vul-projects" - description = "" - - @extend_schema( - deprecated=True, - summary="获取组件漏洞项目列表", - tags=["Project"], - ) - def get(self, request, vul_id): - try: - auth_users = self.get_auth_users(request.user) - asset_queryset = self.get_auth_assets(auth_users) - asset_vul = IastAssetVul.objects.filter(id=vul_id).first() - if not asset_vul: - return R.failure(msg=_('Components of the vul do not exist or no permission to access')) - - package_kw = request.query_params.get('keyword', None) - - if package_kw and package_kw.strip() != '': - asset_queryset = asset_queryset.filter(project_name__icontains=package_kw) - - # todo 是否限制只展示4层以内的项目 - asset_queryset = asset_queryset.filter( - iastvulassetrelation__asset_vul_id=vul_id, - project_id__gt=0).values( - 'project_id', - 'project_name', - 'project_version_id', - ).distinct().all() - - data = [] - asset_queryset = asset_queryset.order_by('project_id', 'project_version_id') - page = int(request.query_params.get('page', 1)) - page_size = int(request.query_params.get('pageSize', 10)) - page_summary, page_data = self.get_paginator(asset_queryset, page, page_size) - - if page_data: - for _data in page_data: - project_version_query = IastProjectVersion.objects.filter( - project_id=_data['project_id'], - id=_data['project_version_id']).first() - if project_version_query: - project_version = project_version_query.version_name - else: - project_version = '' - asset_info = asset_queryset.filter( - project_id=_data['project_id']).order_by( - 'level_id').values('level_id').first() - level = IastVulLevel.objects.filter( - id=asset_info['level_id']).first() - #level = IastVulLevel.objects.filter(id=_data['level_id']).first() - level_name = level.name_value if level else "" - data.append({ - 'project_id': - _data['project_id'], - 'project_name': - _data['project_name'], - 'level': - level_name, - 'project_version': - project_version, - 'dependency_level': - 0, - 'project_version_id': - _data['project_version_id'] - }) - - return R.success(data=data, page=page_summary) - except Exception as e: - logger.error(e) - return R.failure(msg=_('Component vul projects query failed')) - - -def get_tree(dep_list: List[str]): - a = {} - len_of_list = len(dep_list) - for ind, i in enumerate(dep_list): - if a: - a = {"package_name": i, "dependency_asset": [a], "dependency_level": len_of_list - ind} - else: - a = { - "package_name": i, - "dependency_level": len_of_list - ind, - } - return a - -class ProjectsAssets(UserEndPoint): - name = "api-v1-sca-vul-project-assets" - description = "" - - @extend_schema( - deprecated=True, - summary="获取项目组件列表", - tags=["Project"], - ) - def get(self, request): - - try: - auth_users = self.get_auth_users(request.user) - asset_queryset = self.get_auth_assets(auth_users) - - vul_id = request.query_params.get('vul_id', 0) - project_id = request.query_params.get('project_id', 0) - project_version_id = request.query_params.get('project_version_id', 0) - - if not project_id: - return R.failure(msg=_('Param error')) - - asset_vul = IastAssetVul.objects.filter(id=vul_id).first() - if not asset_vul: - return R.failure(msg=_('Vul not exist')) - - # 当前漏洞所在组件 - asset_queryset = asset_queryset.filter( - iastvulassetrelation__asset_vul_id=vul_id, - project_id=project_id, - project_version_id=project_version_id).values( - 'id', 'dependency_level', 'package_name', 'version', - 'level', 'parent_dependency_id', - 'iastvulassetrelation__vul_dependency_path').order_by( - 'level_id').first() - if asset_queryset['iastvulassetrelation__vul_dependency_path'] is not None: - return R.success(data=[get_tree(asset_queryset['iastvulassetrelation__vul_dependency_path'])]) - dependency_level = asset_queryset['dependency_level'] - parent_dependency_id = asset_queryset['parent_dependency_id'] - LEVEL_MAPS = dict() - if asset_queryset['level'] not in LEVEL_MAPS: - asset_level = IastVulLevel.objects.filter(id=asset_queryset['level']).values('id', - 'name_value').first() - else: - asset_level = LEVEL_MAPS[asset_queryset['level']] - asset_queryset['level_id'] = asset_level['id'] - asset_queryset['level'] = asset_level['name_value'] - asset_queryset['dependency_asset'] = [] - - resp_data = asset_queryset - asset_queryset_dependency = dict() - if dependency_level > 1 and parent_dependency_id > 0: - dependency_asset = _get_parent_dependency(asset_queryset_dependency, parent_dependency_id) - if dependency_asset: - for dependency_key in dependency_asset: - if dependency_asset[dependency_key]['level'] not in LEVEL_MAPS: - asset_level = IastVulLevel.objects.filter( - id=dependency_asset[dependency_key]['level']).values('id', 'name_value').first() - else: - asset_level = LEVEL_MAPS[asset_queryset['level']] - - dependency_asset[dependency_key]['level_id'] = asset_level['id'] - dependency_asset[dependency_key]['level'] = asset_level['name_value'] - - if dependency_key + 1 not in dependency_asset: - dependency_asset[dependency_key]['dependency_asset'] = [asset_queryset] - else: - dependency_key_n = dependency_key + 1 - dependency_asset[dependency_key]['dependency_asset'] = [dependency_asset[dependency_key_n]] - resp_data = dependency_asset[1] - - return R.success(data=[resp_data]) - except Exception as e: - logger.error(e, exc_info=True) - return R.failure(msg=_('Component vul projects query failed')) - - -def _get_parent_dependency(asset_queryset_dependency, parent_dependency_id): - parent_asset = Asset.objects.filter(id=parent_dependency_id).values('id', 'dependency_level', 'package_name', - 'version', 'level', - 'parent_dependency_id').first() - if parent_asset: - dependency_level = parent_asset['dependency_level'] - parent_dependency_id = parent_asset['parent_dependency_id'] - asset_queryset_dependency[dependency_level] = parent_asset - if dependency_level > 1 and parent_dependency_id > 0: - _get_parent_dependency(asset_queryset_dependency, parent_dependency_id) - - return asset_queryset_dependency diff --git a/dongtai_web/dongtai_sca/views/newpackage.py b/dongtai_web/dongtai_sca/views/newpackage.py index cee9a2558..7e723ff7d 100644 --- a/dongtai_web/dongtai_sca/views/newpackage.py +++ b/dongtai_web/dongtai_sca/views/newpackage.py @@ -12,7 +12,6 @@ from dongtai_web.utils import extend_schema_with_envcheck_v2, get_response_serializer from rest_framework import serializers -from dongtai_web.dongtai_sca.utils import get_asset_id_by_aggr_id from dongtai_common.models.assetv2 import AssetV2, AssetV2Global from rest_framework.serializers import ValidationError from dongtai_common.serializers.assetv2 import PackeageScaAssetDetailSerializer diff --git a/dongtai_web/dongtai_sca/views/newpackagedetail.py b/dongtai_web/dongtai_sca/views/newpackagedetail.py index 8945cb8ea..7ed3917e4 100644 --- a/dongtai_web/dongtai_sca/views/newpackagedetail.py +++ b/dongtai_web/dongtai_sca/views/newpackagedetail.py @@ -11,7 +11,6 @@ from dongtai_web.utils import extend_schema_with_envcheck_v2, get_response_serializer from rest_framework import serializers -from dongtai_web.dongtai_sca.utils import get_asset_id_by_aggr_id from dongtai_common.models.assetv2 import ( AssetV2, AssetV2Global, diff --git a/dongtai_web/dongtai_sca/views/newpackageprojects.py b/dongtai_web/dongtai_sca/views/newpackageprojects.py index 9dbd5c63d..dc2725084 100644 --- a/dongtai_web/dongtai_sca/views/newpackageprojects.py +++ b/dongtai_web/dongtai_sca/views/newpackageprojects.py @@ -13,7 +13,6 @@ from rest_framework.serializers import ValidationError from django.db.models import Q, F, Value -from dongtai_web.dongtai_sca.utils import get_asset_id_by_aggr_id from dongtai_common.models.assetv2 import AssetV2, AssetV2Global from dongtai_common.models.project import IastProject from rest_framework_dataclasses.serializers import DataclassSerializer diff --git a/dongtai_web/dongtai_sca/views/newpackageprojectversions.py b/dongtai_web/dongtai_sca/views/newpackageprojectversions.py index 94a595fb4..acbddb884 100644 --- a/dongtai_web/dongtai_sca/views/newpackageprojectversions.py +++ b/dongtai_web/dongtai_sca/views/newpackageprojectversions.py @@ -11,7 +11,6 @@ from dongtai_web.utils import extend_schema_with_envcheck_v2, get_response_serializer from rest_framework import serializers -from dongtai_web.dongtai_sca.utils import get_asset_id_by_aggr_id from dongtai_common.models.assetv2 import AssetV2, AssetV2Global from dongtai_common.models.project import IastProject from rest_framework_dataclasses.serializers import DataclassSerializer diff --git a/dongtai_web/dongtai_sca/views/newpackagesummary.py b/dongtai_web/dongtai_sca/views/newpackagesummary.py index f00a194ef..afae8023e 100644 --- a/dongtai_web/dongtai_sca/views/newpackagesummary.py +++ b/dongtai_web/dongtai_sca/views/newpackagesummary.py @@ -12,7 +12,6 @@ from rest_framework import serializers from rest_framework.serializers import ValidationError -from dongtai_web.dongtai_sca.utils import get_asset_id_by_aggr_id from dongtai_common.models.assetv2 import AssetV2, AssetV2Global, IastAssetLicense from rest_framework_dataclasses.serializers import DataclassSerializer from dataclasses import dataclass, field diff --git a/dongtai_web/dongtai_sca/views/newpackagevuldetail.py b/dongtai_web/dongtai_sca/views/newpackagevuldetail.py index a759511dc..9f72c76b0 100644 --- a/dongtai_web/dongtai_sca/views/newpackagevuldetail.py +++ b/dongtai_web/dongtai_sca/views/newpackagevuldetail.py @@ -11,7 +11,6 @@ from dongtai_web.utils import extend_schema_with_envcheck_v2, get_response_serializer from rest_framework import serializers -from dongtai_web.dongtai_sca.utils import get_asset_id_by_aggr_id from dongtai_common.models.asset_vul_v2 import IastAssetVulV2 from dongtai_common.serializers.assetvulv2 import PackageVulSerializer diff --git a/dongtai_web/dongtai_sca/views/newpackagevullevel.py b/dongtai_web/dongtai_sca/views/newpackagevullevel.py index 817dd4596..11d377f1b 100644 --- a/dongtai_web/dongtai_sca/views/newpackagevullevel.py +++ b/dongtai_web/dongtai_sca/views/newpackagevullevel.py @@ -11,7 +11,6 @@ from dongtai_web.utils import extend_schema_with_envcheck_v2, get_response_serializer from rest_framework import serializers -from dongtai_web.dongtai_sca.utils import get_asset_id_by_aggr_id from dongtai_common.models.asset_vul_v2 import IastAssetVulV2 from rest_framework_dataclasses.serializers import DataclassSerializer from dongtai_web.dongtai_sca.scan.utils import get_level diff --git a/dongtai_web/dongtai_sca/views/newpackagevuls.py b/dongtai_web/dongtai_sca/views/newpackagevuls.py index 462bcde00..06706e80e 100644 --- a/dongtai_web/dongtai_sca/views/newpackagevuls.py +++ b/dongtai_web/dongtai_sca/views/newpackagevuls.py @@ -12,7 +12,6 @@ from rest_framework import serializers from rest_framework.serializers import ValidationError -from dongtai_web.dongtai_sca.utils import get_asset_id_by_aggr_id from dongtai_common.models.asset_vul_v2 import IastAssetVulV2 from dongtai_common.serializers.assetvulv2 import PackageVulSerializer from dataclasses import dataclass, field diff --git a/dongtai_web/dongtai_sca/views/package.py b/dongtai_web/dongtai_sca/views/package.py deleted file mode 100644 index bf1dc510f..000000000 --- a/dongtai_web/dongtai_sca/views/package.py +++ /dev/null @@ -1,80 +0,0 @@ -import logging - -from dongtai_common.models import User -from dongtai_web.dongtai_sca.models import Package -from django.http import JsonResponse -from rest_framework import views -from django.core.paginator import Paginator -from django.forms.models import model_to_dict -from dongtai_common.endpoint import R, AnonymousAndUserEndPoint, UserEndPoint -from django.utils.translation import gettext_lazy as _ -from drf_spectacular.utils import extend_schema - -from dongtai_web.dongtai_sca.utils import get_asset_id_by_aggr_id - -logger = logging.getLogger(__name__) - - -class PackageList(AnonymousAndUserEndPoint): - - @extend_schema( - tags=[_('Component')], - summary="组件列表", - deprecated=True, - ) - def get(self, request): - filter_fields = ['hash', 'aql', 'ecosystem', 'name', 'version'] - _filter = Package.objects.filter().order_by("-updated_at") - kwargs = {} - for filter_field in filter_fields: - _val = request.GET.get(filter_field, "") - if _val != "": - kwargs[filter_field] = request.GET.get(filter_field, "") - _filter = _filter.filter(**kwargs) - - page = int(request.GET.get("page", 1)) - page_size = int(request.GET.get("page_size", 5)) - - pageinfo = Paginator(_filter, per_page=page_size) - result = { - 'data': [], - 'msg': 'success', - 'page': { - 'alltotal': pageinfo.count, - 'num_pages': pageinfo.num_pages, - 'page_size': pageinfo.per_page, - }, - 'status': 201 - } - if page == 0 or page <= pageinfo.num_pages: - rows = pageinfo.page(page).object_list - - for row in rows: - result['data'].append(model_to_dict(row)) - - return JsonResponse(result) - - -class AssetAggrDetailAssetIds(UserEndPoint): - name = "api-v1-sca-aggr-assets" - description = "" - - @extend_schema( - tags=[_('Component')], - summary="组件详情", - deprecated=True, - ) - def get(self, request, aggr_id): - try: - auth_users = self.get_auth_users(request.user) - asset_queryset = self.get_auth_assets(auth_users) - - asset_ids = [] - for asset in asset_queryset: - asset_ids.append(asset.id) - asset_ids = get_asset_id_by_aggr_id(aggr_id, asset_ids) - - return R.success(data=asset_ids) - except Exception as e: - logger.error(e) - return R.failure(msg=_('Component asset id query failed')) diff --git a/dongtai_web/serializers/vul.py b/dongtai_web/serializers/vul.py index 554112b9a..5aeeda054 100644 --- a/dongtai_web/serializers/vul.py +++ b/dongtai_web/serializers/vul.py @@ -36,9 +36,23 @@ class VulSerializer(serializers.ModelSerializer): class Meta: model = IastVulnerabilityModel fields = [ - 'id', 'type', 'hook_type_id', 'url', 'uri', 'agent_id', 'level_id', - 'http_method', 'top_stack', 'bottom_stack', 'taint_position', - 'latest_time', 'first_time', 'language', 'status' + 'id', + 'type', + 'hook_type_id', + 'url', + 'uri', + 'agent_id', + 'level_id', + 'http_method', + 'top_stack', + 'bottom_stack', + 'taint_position', + 'latest_time', + 'first_time', + 'language', + 'status', + 'header_vul_urls', + 'is_header_vul', ] @staticmethod diff --git a/dongtai_web/urls.py b/dongtai_web/urls.py index 66d0df8cd..03eb055dd 100644 --- a/dongtai_web/urls.py +++ b/dongtai_web/urls.py @@ -46,7 +46,6 @@ from dongtai_web.views.engine_hook_rule_types import EngineHookRuleTypesEndPoint from dongtai_web.views.engine_hook_rules import EngineHookRulesEndPoint from dongtai_web.views.engine_method_pool_detail import MethodPoolDetailProxy -from dongtai_web.views.engine_method_pool_sca import EngineMethodPoolSca from dongtai_web.views.engine_method_pool_search import MethodPoolSearchProxy from dongtai_web.views.log_clear import LogClear from dongtai_web.views.log_delete import LogDelete @@ -242,7 +241,6 @@ path('engine/method_pool/search', MethodPoolSearchProxy.as_view()), path('engine/method_pool/detail', MethodPoolDetailProxy.as_view()), path('engine/method_pool/timerange', MethodPoolTimeRangeProxy.as_view()), - path('engine/method_pool/sca', EngineMethodPoolSca.as_view()), path('engine/graph', MethodGraph.as_view()), path('engine/request/replay', RequestReplayEndPoint.as_view()), path('engine/hook/rule/summary', EngineHookRuleSummaryEndPoint.as_view()), diff --git a/static/i18n/en/LC_MESSAGES/django.po b/static/i18n/en/LC_MESSAGES/django.po index 8bd01f9e1..d96c8d252 100644 --- a/static/i18n/en/LC_MESSAGES/django.po +++ b/static/i18n/en/LC_MESSAGES/django.po @@ -3040,6 +3040,10 @@ msgstr "Current version stopped for maintenance or it is not a secure version" msgid "Component information query failed" msgstr "Component information query failed" +#: sca/views/asset_projects.py:42 +msgid "Component projects query failed" +msgstr "Component projects query failed" + #: iast/views/sca_sidebar_index.py:40 iast/views/sca_summary.py:91 #: iast/views/scas.py:80 iast/views/vul_list_for_plugin.py:54 #: iast/views/vul_sidebar_index.py:70 iast/views/vul_summary.py:96 diff --git a/static/i18n/zh/LC_MESSAGES/django.po b/static/i18n/zh/LC_MESSAGES/django.po index 2e3d3c2a4..634323b6e 100644 --- a/static/i18n/zh/LC_MESSAGES/django.po +++ b/static/i18n/zh/LC_MESSAGES/django.po @@ -2788,6 +2788,10 @@ msgstr "通过指定id来获取对应组件的详情" msgid "Components do not exist or no permission to access" msgstr "组件不存在或无权限访问" +#: sca/views/asset_projects.py:42 +msgid "Component projects query failed" +msgstr "组件关联项目获取失败" + #: iast/views/sca_details.py:137 msgid "Current version stopped for maintenance or it is not a secure version" msgstr "当前版本已停止维护或暂无安全版本" From 0af074aca7470a97d4b2b769cc66b0f7fdbdf6d4 Mon Sep 17 00:00:00 2001 From: st1020 Date: Thu, 6 Jul 2023 12:28:29 +0800 Subject: [PATCH 060/161] feat: remove unused files --- dongtai_web/views/engine_method_pool_sca.py | 63 --------------------- 1 file changed, 63 deletions(-) delete mode 100644 dongtai_web/views/engine_method_pool_sca.py diff --git a/dongtai_web/views/engine_method_pool_sca.py b/dongtai_web/views/engine_method_pool_sca.py deleted file mode 100644 index 27c96f446..000000000 --- a/dongtai_web/views/engine_method_pool_sca.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -# -*- coding:utf-8 -*- -# author: owefsad@huoxian.cn -# project: dongtai-webapi -from dongtai_common.endpoint import AnonymousAndUserEndPoint, R -from dongtai_common.models.agent_method_pool import MethodPool -from dongtai_common.models.asset import Asset - -from dongtai_web.serializers.sca import ScaSerializer -from django.utils.translation import gettext_lazy as _ -from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer -from dongtai_common.models.sca_maven_db import ScaMavenDb - -_ResponseSerializer = get_response_serializer( - data_serializer=ScaSerializer(many=True), ) - - -class EngineMethodPoolSca(AnonymousAndUserEndPoint): - @extend_schema_with_envcheck( - [{ - 'name': "method_pool_id", - 'type': int, - 'required': True - }], - tags=[_('Method Pool')], - summary=_('Method Pool Component'), - description=_("Get the component information list of the tainted call chain."), - response_schema=_ResponseSerializer, - ) - def get(self, request): - method_pool_id = request.query_params.get('method_pool_id') - - if method_pool_id is None: - return R.failure(msg=_('method_pool_id is empty')) - - method_pool = MethodPool.objects.filter( - id=method_pool_id).values('agent_id').first() - if method_pool is None: - return R.failure(msg=_('method_pool does not exist')) - - agent_id = method_pool['agent_id'] - auth_agents = self.get_auth_and_anonymous_agents(request.user) - if auth_agents is None or auth_agents.filter( - id=agent_id).values('id').exists() is False: - return R.failure(msg=_('method_pool has no permission')) - - project_data = auth_agents.filter(id=agent_id).values( - 'bind_project_id', 'project_version_id').first() - project_id = project_data['bind_project_id'] - project_version_id = project_data['project_version_id'] - - queryset = Asset.objects.filter(agent_id=agent_id) - license_dict = { - i['sha_1']: i['license'] - for i in ScaMavenDb.objects.filter(sha_1__in=queryset.values( - 'signature_value')).values('license', 'sha_1') - } - return R.success( - data=ScaSerializer(queryset.select_related('level', 'agent'), - context={ - 'license_dict': license_dict - }, - many=True).data) From 08bf24ad55a872caaedfda383a7c5d890dda17e0 Mon Sep 17 00:00:00 2001 From: st1020 Date: Thu, 6 Jul 2023 14:46:27 +0800 Subject: [PATCH 061/161] feat: add monkeypatch --- dongtai_web/aggr_vul/aggr_vul_list.py | 16 +++++++++++++--- dongtai_web/aggr_vul/app_vul_list.py | 5 +++++ dongtai_web/views/vuls.py | 7 ++++++- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/dongtai_web/aggr_vul/aggr_vul_list.py b/dongtai_web/aggr_vul/aggr_vul_list.py index 85deec3ab..b6f209d6d 100644 --- a/dongtai_web/aggr_vul/aggr_vul_list.py +++ b/dongtai_web/aggr_vul/aggr_vul_list.py @@ -1,4 +1,5 @@ # 按类型获取 组件漏洞 应用漏洞列表 +from typing import Any from elasticsearch_dsl import Q, Search from dongtai_common.models.asset_vul import IastAssetVulnerabilityDocument from dongtai_common.common.utils import make_hash @@ -266,9 +267,10 @@ def post(self, request): availability_str, # "type_name": item.type_name, } - cwe = get_cve_from_cve_nums(cur_data["vul_cve_nums"]) - if cwe: - cur_data['vul_cve_nums']['cwe_num'] = cwe + if cur_data["vul_cve_nums"]: + cwe = get_cve_from_cve_nums(cur_data["vul_cve_nums"]) + if cwe: + cur_data['vul_cve_nums']['cwe_num'] = cwe vul_ids.append(item.id) content_list.append(cur_data) # 追加 用户 权限 @@ -318,6 +320,7 @@ def post(self, request): for row in content_list: row["pro_info"] = pro_arr.get(row['id'], []) row['type_name'] = ",".join(type_arr.get(row['id'], [])) + set_vul_inetration(content_list, vul_ids, request.user.id) return R.success(data={ 'messages': content_list, 'page': { @@ -327,6 +330,13 @@ def post(self, request): }, ) +def set_vul_inetration( + content_list: list[dict[str, Any]], + vul_ids: list[int], + user_id: int, +) -> None: + pass + def get_vul_list_from_elastic_search(user_id, project_ids=[], project_version_ids=[], diff --git a/dongtai_web/aggr_vul/app_vul_list.py b/dongtai_web/aggr_vul/app_vul_list.py index 0941665a0..d042d5005 100644 --- a/dongtai_web/aggr_vul/app_vul_list.py +++ b/dongtai_web/aggr_vul/app_vul_list.py @@ -1,3 +1,4 @@ +from typing import Any from rest_framework.serializers import ValidationError from dongtai_common.endpoint import R from dongtai_common.endpoint import UserEndPoint @@ -218,6 +219,7 @@ def post(self, request): for i in end['data']: i['status__name'] = status_obj.get(i['status_id'], "") + set_vul_inetration(end, request.user.id) return R.success(data={ 'messages': end['data'], 'page': { @@ -226,6 +228,9 @@ def post(self, request): } }, ) +def set_vul_inetration(end: dict[str, Any], user_id: int) -> None: + pass + def get_vul_list_from_elastic_search(departments, project_ids=[], diff --git a/dongtai_web/views/vuls.py b/dongtai_web/views/vuls.py index 45432396a..c0440f96c 100644 --- a/dongtai_web/views/vuls.py +++ b/dongtai_web/views/vuls.py @@ -3,7 +3,7 @@ # author:owefsad # software: PyCharm # project: lingzhi-webapi - +from typing import Any from dongtai_common.endpoint import R from dongtai_common.endpoint import UserEndPoint from dongtai_common.models.vul_level import IastVulLevel @@ -316,4 +316,9 @@ def get(self, request): item['level'] = allTypeArr.get(item['level_id'], "") end['data'].append(item) end['page'] = page_summary + set_vul_inetration(end, request.user.id) return R.success(page=page_summary, data=end['data']) + + +def set_vul_inetration(end: dict[str, Any], user_id: int) -> None: + pass From 95e0695a8458713ef54069c4f7dffad1da6090fd Mon Sep 17 00:00:00 2001 From: st1020 Date: Thu, 6 Jul 2023 17:40:33 +0800 Subject: [PATCH 062/161] feat: remove project template --- dongtai_web/projecttemplate/base.py | 163 ---------------------------- dongtai_web/urls.py | 13 --- 2 files changed, 176 deletions(-) delete mode 100644 dongtai_web/projecttemplate/base.py diff --git a/dongtai_web/projecttemplate/base.py b/dongtai_web/projecttemplate/base.py deleted file mode 100644 index 926c0f61e..000000000 --- a/dongtai_web/projecttemplate/base.py +++ /dev/null @@ -1,163 +0,0 @@ -from dongtai_common.models.project import ( - IastProject, - IastProjectTemplate, - VulValidation, -) -from rest_framework import serializers -from dongtai_common.endpoint import UserEndPoint, R, TalentAdminEndPoint -from rest_framework import viewsets -from django.utils.translation import gettext_lazy as _ -from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer -from dongtai_common.permissions import UserPermission, ScopedPermission, SystemAdminPermission, TalentAdminPermission - - -class PaginationSerializer(serializers.Serializer): - page_size = serializers.IntegerField(default=20, - help_text=_('Number per page')) - page = serializers.IntegerField(default=1, help_text=_('Page index')) - - -class ProjectTemplateCreateArgsSerializer(serializers.Serializer): - id = serializers.IntegerField( - min_value=1, - help_text=_("The id corresponding to the scanning strategy."), - required=False) - template_name = serializers.CharField(help_text=_('The name of project')) - scan_id = serializers.IntegerField( - min_value=1, - max_value=2**32, - help_text=_("The id corresponding to the scanning strategy.")) - vul_validation = serializers.IntegerField( - min_value=0, - max_value=2, - help_text="vul validation switch, 0-FOLLOW_GLOBAL, 1-ENABLE,2-DISABLE") - data_gather = serializers.JSONField(help_text="data gather settings", - required=False) - data_gather_is_followglobal = serializers.IntegerField(required=False, - min_value=0, - max_value=2, - default=0) - blacklist_is_followglobal = serializers.IntegerField(required=False, - min_value=0, - max_value=2, - default=0) - blacklist = serializers.SerializerMethodField(required=False) - is_system = serializers.IntegerField(required=False, default=0) - - def get_blacklist(self, obj): - return [] - - class Meta: - model = IastProjectTemplate - fields = [ - 'id', - 'template_name', - 'scan_id', - 'vul_validation', - 'data_gather', - 'data_gather_is_followglobal', - 'blacklist_is_followglobal', - 'blacklist', - ] - - -def template_create(data, user): - data['user_id'] = user.id - for field in ["blacklist"]: - if field in data: - del data[field] - project_template = IastProjectTemplate.objects.create(**data) - pk = project_template.id - return pk - - -def template_update(pk, data, user): - data['user_id'] = user.id - for field in ["blacklist"]: - if field in data: - del data[field] - IastProjectTemplate.objects.filter(pk=pk).update(**data) - - -class IastProjectTemplateView(TalentAdminEndPoint, viewsets.ViewSet): - name = "api-v1-agent-project-template" - description = _("project_template") - - permission_classes_by_action = {'list': (UserPermission, )} - - def get_permissions(self): - try: - return [ - permission() for permission in - self.permission_classes_by_action[self.action] - ] - except KeyError: - return [permission() for permission in self.permission_classes] - - @extend_schema_with_envcheck(request=ProjectTemplateCreateArgsSerializer, - summary=_('Create project template'), - description=_("Create project template"), - tags=[_('projectemplate')]) - def create(self, request): - ser = ProjectTemplateCreateArgsSerializer(data=request.data) - ser.is_valid() - if ser.errors: - return R.failure(data=ser.errors) - data = {} - for field in ProjectTemplateCreateArgsSerializer.Meta.fields: - if field in request.data and field != 'id': - data[field] = request.data[field] - template_create(data, request.user) - return R.success() - - @extend_schema_with_envcheck(request=ProjectTemplateCreateArgsSerializer, - summary=_('Update project template'), - description=_("Update project template"), - tags=[_('projectemplate')]) - def update(self, request, pk): - ser = ProjectTemplateCreateArgsSerializer(data=request.data) - ser.is_valid() - if ser.errors: - return R.failure(data=ser.errors) - data = {} - for field in ProjectTemplateCreateArgsSerializer.Meta.fields: - if field in request.data and field != 'id': - data[field] = request.data[field] - template_update(pk, data, request.user) - return R.success() - - @extend_schema_with_envcheck([PaginationSerializer], - summary=_('List project template'), - description=_("List project template"), - tags=[_('projectemplate')]) - def list(self, request): - ser = PaginationSerializer(data=request.GET) - ser.is_valid() - if ser.errors: - return R.failure(data=ser.errors) - page_size = ser.validated_data['page_size'] - page = ser.validated_data['page'] - summary, templates = self.get_paginator( - IastProjectTemplate.objects.values().order_by( - '-latest_time').all(), page, page_size) - return R.success( - data=ProjectTemplateCreateArgsSerializer(templates, - many=True).data, - page=summary, - ) - - @extend_schema_with_envcheck(summary=_('delete project template'), - description=_("delete project template"), - tags=[_('projectemplate')]) - def delete(self, request, pk): - IastProjectTemplate.objects.filter(pk=pk).delete() - return R.success() - - @extend_schema_with_envcheck(summary=_('get project template'), - description=_("get project template"), - tags=[_('projectemplate')]) - def retrieve(self, request, pk): - obj = IastProjectTemplate.objects.filter(pk=pk).values().first() - if not obj: - return R.failure() - return R.success(data=ProjectTemplateCreateArgsSerializer(obj).data) diff --git a/dongtai_web/urls.py b/dongtai_web/urls.py index 03eb055dd..99ca954ec 100644 --- a/dongtai_web/urls.py +++ b/dongtai_web/urls.py @@ -152,7 +152,6 @@ from dongtai_web.vul_log.vul_log_view import VulLogViewSet from dongtai_web.vul_recheck_payload.vul_recheck_payload import VulReCheckPayloadViewSet from dongtai_web.header_vul.base import HeaderVulViewSet -from dongtai_web.projecttemplate.base import IastProjectTemplateView from dongtai_web.dast.webhook import DastWebhook from dongtai_web.dast.page import DastVulsEndPoint from dongtai_web.dast.manage import DastManageEndPoint @@ -378,18 +377,6 @@ path('header_vul/', HeaderVulViewSet.as_view({ 'delete': "delete", })), - path( - 'projecttemplate/', - IastProjectTemplateView.as_view({ - 'get': "retrieve", - 'put': 'update', - 'delete': 'delete', - })), - path('projecttemplate', - IastProjectTemplateView.as_view({ - 'get': "list", - 'post': 'create', - })), path('dast_webhook', DastWebhook.as_view()), path('dastvul/', DastVulsEndPoint.as_view({ 'get': "single", From b13acc356826cc1d20ce41a8113b89759cb8e186 Mon Sep 17 00:00:00 2001 From: st1020 Date: Thu, 6 Jul 2023 17:52:41 +0800 Subject: [PATCH 063/161] feat: remove data clean --- dongtai_conf/celery.py | 1 - dongtai_engine/plugins/data_clean.py | 97 ---------------------- dongtai_web/systemmonitor/data_clean.py | 106 ------------------------ dongtai_web/systemmonitor/urls.py | 6 -- 4 files changed, 210 deletions(-) delete mode 100644 dongtai_engine/plugins/data_clean.py delete mode 100644 dongtai_web/systemmonitor/data_clean.py diff --git a/dongtai_conf/celery.py b/dongtai_conf/celery.py index a70be8587..9232a67a0 100644 --- a/dongtai_conf/celery.py +++ b/dongtai_conf/celery.py @@ -83,7 +83,6 @@ "dongtai_engine.tasks.clear_error_log": {'exchange': 'dongtai-periodic-task', 'routing_key': 'dongtai-periodic-task'}, "dongtai_engine.tasks.vul_recheck": {'exchange': 'dongtai-periodic-task', 'routing_key': 'dongtai-periodic-task'}, "dongtai_engine.preheat.function_preheat": {'exchange': 'dongtai-periodic-task', 'routing_key': 'dongtai-periodic-task'}, - "dongtai_engine.plugins.data_clean": {'exchange': 'dongtai-periodic-task', 'routing_key': 'dongtai-periodic-task'}, "dongtai_engine.plugins.project_status": {'exchange': 'dongtai-periodic-task', 'routing_key': 'dongtai-periodic-task'}, } configs["CELERY_ENABLE_UTC"] = False diff --git a/dongtai_engine/plugins/data_clean.py b/dongtai_engine/plugins/data_clean.py deleted file mode 100644 index d370bda6d..000000000 --- a/dongtai_engine/plugins/data_clean.py +++ /dev/null @@ -1,97 +0,0 @@ -from dongtai_common.models.agent import IastAgent -from celery import shared_task -from dongtai_common.models.agent_method_pool import MethodPool -from time import time -from celery.apps.worker import logger -from dongtai_conf.settings import ELASTICSEARCH_STATE -from asgiref.sync import sync_to_async -import asyncio -from typing import List, Tuple -#import asyncio_gevent - -DELETE_BATCH_SIZE = 10000 - - -def chunked_queryset(queryset, chunk_size): - """ Slice a queryset into chunks. """ - - start_pk = 0 - queryset = queryset.order_by('pk') - - while True: - # No entry left - if not queryset.filter(pk__gt=start_pk).exists(): - break - - try: - # Fetch chunk_size entries if possible - end_pk = queryset.filter(pk__gt=start_pk).values_list( - 'pk', flat=True)[chunk_size - 1] - - # Fetch rest entries if less than chunk_size left - except IndexError: - end_pk = queryset.values_list('pk', flat=True).last() - - yield queryset.filter(pk__gt=start_pk).filter(pk__lte=end_pk) - - start_pk = end_pk - - -@shared_task(queue='dongtai-periodic-task', - time_limit=60 * 60 * 2, - soft_time_limit=60 * 60 * 4) -def data_cleanup(days: int): - delete_time_stamp = int(time()) - 60 * 60 * 24 * days - if ELASTICSEARCH_STATE: - # use delete to trigger the signal to delete related elasticsearch doc - qs = MethodPool.objects.filter(update_time__lte=delete_time_stamp) - for i in chunked_queryset(qs, DELETE_BATCH_SIZE): - i.delete() - else: - # use _raw_delete to reduce the delete time and memory usage. - # it could aviod to load every instance into memory. - latest_id = MethodPool.objects.filter( - update_time__lte=delete_time_stamp).order_by('-id').values_list( - 'id', flat=True).first() - first_id = MethodPool.objects.filter( - update_time__lte=delete_time_stamp).order_by('id').values_list( - 'id', flat=True).first() - if not any((latest_id, first_id)) or not isinstance(latest_id, int) or not isinstance(first_id, int): - logger.info("no data for clean up") - return - if all((latest_id, first_id)): - batch_clean(latest_id, first_id, 10000) - # qs = MethodPool.objects.filter(pk__lte=latest_id) - # qs._raw_delete(qs.db) - - logger.info(f'Delete offline agent 30 days ago') - agent_latest_id = IastAgent.objects.filter( - latest_time__lte=delete_time_stamp).order_by('-id').values_list( - 'id', flat=True).first() - if any([agent_latest_id]): - IastAgent.objects.filter(pk__lte=agent_latest_id, - online=0).delete() - - -@sync_to_async(thread_sensitive=False) -def data_clean_batch(upper_id: int, lower_id: int): - qs = MethodPool.objects.filter(pk__lt=upper_id, pk__gte=lower_id) - logger.info(f"data cleaning {upper_id}-{lower_id} ") - qs._raw_delete(qs.db) - - -#@asyncio_gevent.async_to_sync -async def loop_main(range_list: List[Tuple[int, int]]): - coros = [ - data_clean_batch(upper_id, lower_id) - for upper_id, lower_id in range_list - ] - await asyncio.gather(*coros) - - -def batch_clean(upper_id: int, lower_id: int, batch_size: int): - chunk_range = list( - zip(range(lower_id + batch_size, upper_id + batch_size, batch_size), - range(lower_id, upper_id, batch_size))) - loop = asyncio.get_event_loop() - loop.run_until_complete(loop_main(chunk_range)) diff --git a/dongtai_web/systemmonitor/data_clean.py b/dongtai_web/systemmonitor/data_clean.py deleted file mode 100644 index ff7c978b3..000000000 --- a/dongtai_web/systemmonitor/data_clean.py +++ /dev/null @@ -1,106 +0,0 @@ -from dongtai_common.endpoint import R -from dongtai_common.endpoint import UserEndPoint -from dongtai_conf.settings import config -from dongtai_common.models.profile import IastProfile -from django.utils.translation import gettext_lazy as _ -from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers -from rest_framework.serializers import ValidationError -from django.forms.models import model_to_dict -import json -from datetime import datetime -from django_celery_beat.models import ( - CrontabSchedule, - PeriodicTask, -) -from dongtai_engine.plugins.data_clean import data_cleanup - - -class DataCleanSettingsSer(serializers.Serializer): - clean_time = serializers.TimeField(format="%H:%M:%S") - days_before = serializers.IntegerField() - enable = serializers.BooleanField() - - -class DataCleanDoItNowArgsSer(serializers.Serializer): - days_before = serializers.IntegerField() - - -class DataCleanEndpoint(UserEndPoint): - - @extend_schema_with_envcheck(summary=_('Get Profile'), - description=_("Get Profile with key"), - tags=[_('Profile')]) - def get(self, request): - key = 'data_clean' - profile = IastProfile.objects.filter(key=key).values_list( - 'value', flat=True).first() - if profile is None: - return R.failure( - msg=_("Failed to get {} configuration").format(key)) - data = json.loads(profile) - return R.success(data=data) - - @extend_schema_with_envcheck(summary=_('Profile modify'), - request=DataCleanSettingsSer, - description=_("Modifiy Profile with key"), - tags=[_('Profile')]) - def post(self, request): - ser = DataCleanSettingsSer(data=request.data) - try: - if ser.is_valid(True): - pass - except ValidationError as e: - return R.failure(data=e.detail) - key = 'data_clean' - datetime_obj = datetime.strptime(ser.data['clean_time'], '%H:%M:%S') - hour = datetime_obj.hour - minute = datetime_obj.hour - enabled = 1 if ser.data['enable'] else 0 - kwargs = {'days': int(ser.data['days_before'])} - schedule, _ = CrontabSchedule.objects.get_or_create( - minute=minute, - hour=hour, - day_of_week='*', - day_of_month='*', - month_of_year='*', - ) - PeriodicTask.objects.update_or_create( - name='data clean functions', # simply describes this periodic task. - defaults={ - 'crontab': schedule, # we created this above. - 'enabled': enabled, - 'task': - 'dongtai_engine.plugins.data_clean.data_cleanup', # name of task. - 'args': json.dumps([]), - 'kwargs': json.dumps(kwargs), - }) - value = json.dumps(ser.data) - try: - obj, created = IastProfile.objects.update_or_create( - { - 'key': key, - 'value': value - }, key=key) - except Exception as e: - return R.failure(msg=_("Update {} failed").format(key)) - data = json.loads(value) - return R.success(data=data) - - -class DataCleanDoItNowEndpoint(UserEndPoint): - - @extend_schema_with_envcheck(summary=_('Get Profile'), - request=DataCleanDoItNowArgsSer, - description=_("Get Profile with key"), - tags=[_('Profile')]) - def post(self, request): - ser = DataCleanDoItNowArgsSer(data=request.data) - try: - if ser.is_valid(True): - pass - except ValidationError as e: - return R.failure(data=e.detail) - data_cleanup.delay(days=ser.data['days_before']) - return R.success() diff --git a/dongtai_web/systemmonitor/urls.py b/dongtai_web/systemmonitor/urls.py index 756e1c4dd..dedd3ba1b 100644 --- a/dongtai_web/systemmonitor/urls.py +++ b/dongtai_web/systemmonitor/urls.py @@ -1,17 +1,11 @@ from django.urls import include, path from rest_framework import routers -from dongtai_web.systemmonitor.data_clean import ( - DataCleanEndpoint, - DataCleanDoItNowEndpoint, -) from dongtai_web.systemmonitor.project_warning import ProjectWarningEndpoint router = routers.DefaultRouter() base_urlpatterns = [ - path("data_clean", DataCleanEndpoint.as_view()), - path("data_clean/task", DataCleanDoItNowEndpoint.as_view()), path("project_warning", ProjectWarningEndpoint.as_view()), ] From 62e026b693076b697d15a4936dce84b4b37c16af Mon Sep 17 00:00:00 2001 From: st1020 Date: Thu, 6 Jul 2023 18:12:58 +0800 Subject: [PATCH 064/161] feat: remove scan strategy --- dongtai_web/urls.py | 20 -- dongtai_web/views/scan_strategys.py | 269 -------------------------- test/iast/views/test_scan_strategy.py | 24 --- 3 files changed, 313 deletions(-) delete mode 100644 dongtai_web/views/scan_strategys.py delete mode 100644 test/iast/views/test_scan_strategy.py diff --git a/dongtai_web/urls.py b/dongtai_web/urls.py index 99ca954ec..4a0351a7c 100644 --- a/dongtai_web/urls.py +++ b/dongtai_web/urls.py @@ -127,11 +127,6 @@ SensitiveInfoRuleBatchView, SensitiveInfoRuleAllView, ) -from dongtai_web.views.scan_strategys import ( - ScanStrategyViewSet, - ScanStrategyRelationProject, - ScanStrategyAllView, -) from dongtai_web.views.details_id import (AgentListWithid, ProjectListWithid, ScaListWithid, VulsListWithid) from dongtai_web.views.vul_recheck_v2 import VulReCheckv2 @@ -283,24 +278,9 @@ SensitiveInfoPatternTypeView.as_view()), path('sensitive_info_rule/_validation', SensitiveInfoPatternValidationView.as_view()), - path('scan_strategy', - ScanStrategyViewSet.as_view({ - 'get': 'list', - 'post': 'create' - })), - path( - 'scan_strategy/', - ScanStrategyViewSet.as_view({ - 'get': 'retrieve', - 'put': 'update', - 'delete': 'destory' - })), - path('scan_strategy//relationprojects', - ScanStrategyRelationProject.as_view()), path('sensitive_info_rule/batch_update', SensitiveInfoRuleBatchView.as_view()), path('sensitive_info_rule/all', SensitiveInfoRuleAllView.as_view()), - path('scan_strategy/all', ScanStrategyAllView.as_view()), path('agent/list/ids', AgentListWithid.as_view()), path('vul/list/ids', VulsListWithid.as_view()), path('sca/list/ids', ScaListWithid.as_view()), diff --git a/dongtai_web/views/scan_strategys.py b/dongtai_web/views/scan_strategys.py deleted file mode 100644 index 18d0899e7..000000000 --- a/dongtai_web/views/scan_strategys.py +++ /dev/null @@ -1,269 +0,0 @@ -import logging - -from dongtai_common.endpoint import UserEndPoint, R -from dongtai_common.utils import const - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers -from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer -from django.utils.text import format_lazy -from rest_framework.serializers import ValidationError -from rest_framework import viewsets - -from django.db import models -from dongtai_common.models.strategy_user import IastStrategyUser -from dongtai_common.models.user import User -import time -from django.db.models import Q -from dongtai_common.permissions import TalentAdminPermission -from dongtai_common.models.project import IastProject -from dongtai_web.serializers.project import ProjectSerializer -from dongtai_web.views.utils.commonview import ( - BatchStatusUpdateSerializerView, - AllStatusUpdateSerializerView, -) -from dongtai_common.common.utils import disable_cache -from dongtai_engine.common.queryset import load_sink_strategy - - -logger = logging.getLogger('dongtai-webapi') - - -class ScanStrategySerializer(serializers.ModelSerializer): - content = serializers.SerializerMethodField() - - class Meta: - model = IastStrategyUser - fields = ['id', 'name', 'content', 'user', 'status', 'created_at'] - - def get_content(self, obj): - try: - return [int(i) for i in obj.content.split(',')] - except Exception as e: - print(e) - return [] - - -class _ScanStrategyArgsSerializer(serializers.Serializer): - page_size = serializers.IntegerField(default=20, - help_text=_('Number per page')) - page = serializers.IntegerField(default=1, help_text=_('Page index')) - name = serializers.CharField( - required=False, - help_text=_( - "The name of the item to be searched, supports fuzzy search.")) - - -class _ProjectSerializer(ProjectSerializer): - class Meta: - model = IastProject - fields = ['id', 'name'] - - -class ScanCreateSerializer(serializers.Serializer): - name = serializers.CharField(required=True) - status = serializers.ChoiceField((-1, 0, 1), required=True) - content = serializers.ListField(child=serializers.IntegerField(), - required=True) - - -class _ScanStrategyRelationProjectArgsSerializer(serializers.Serializer): - size = serializers.IntegerField(default=5, - max_value=50, - min_value=1, - required=False, - help_text=_('Number per page')) - - -class ScanStrategyRelationProject(UserEndPoint): - @extend_schema_with_envcheck( - request=ScanCreateSerializer, - tags=[_('ScanStrategy')], - summary=_('ScanStrategy Relation Projects'), - description=_("Get scan strategy relation projects" - ), - ) - def get(self, request, pk): - ser = _ScanStrategyRelationProjectArgsSerializer(data=request.GET) - try: - if ser.is_valid(True): - size = ser.validated_data['size'] - except ValidationError as e: - return R.failure(data=e.detail) - scan_strategy = IastStrategyUser.objects.filter(pk=pk).first() - projects = IastProject.objects.filter( - scan=scan_strategy).order_by('-latest_time')[::size] - return R.success(data=_ProjectSerializer(projects, many=True).data) - - -class ScanStrategyViewSet(UserEndPoint, viewsets.ViewSet): - - permission_classes_by_action = {} - - def get_permissions(self): - try: - return [ - permission() for permission in - self.permission_classes_by_action[self.action] - ] - except KeyError: - return [permission() for permission in self.permission_classes] - - @extend_schema_with_envcheck( - [_ScanStrategyArgsSerializer], - tags=[_('ScanStrategy')], - summary=_('ScanStrategy List'), - description=_("Get the item corresponding to the user, support fuzzy search based on name." - ), - ) - def list(self, request): - ser = _ScanStrategyArgsSerializer(data=request.GET) - try: - if ser.is_valid(True): - name = ser.validated_data.get('name', None) - page = ser.validated_data['page'] - page_size = ser.validated_data['page_size'] - except ValidationError as e: - return R.failure(data=e.detail) - q = ~Q(status=-1) - if name: - q = Q(name__icontains=name) & q - queryset = IastStrategyUser.objects.filter(q).order_by('-created_at') - if name: - queryset = queryset.filter(name__icontains=name) - page_summary, page_data = self.get_paginator(queryset, page, page_size) - return R.success(data=ScanStrategySerializer(page_data, - many=True).data, - page=page_summary) - - @extend_schema_with_envcheck( - request=ScanCreateSerializer, - tags=[_('ScanStrategy')], - summary=_('ScanStrategy Create'), - description=_("Create ScanStrategy" - ), - ) - def create(self, request): - ser = ScanCreateSerializer(data=request.data) - try: - if ser.is_valid(True): - name = ser.validated_data['name'] - content = ser.validated_data['content'] - status = ser.validated_data['status'] - except ValidationError as e: - return R.failure(data=e.detail) - try: - ser.validated_data['content'] = ','.join([str(i) for i in content]) - obj = IastStrategyUser.objects.create(**ser.validated_data, - user=request.user) - return R.success(msg=_('create success'), - data=ScanStrategySerializer(obj).data) - except Exception as e: - logger.error(e) - return R.failure() - - @extend_schema_with_envcheck( - request=ScanCreateSerializer, - tags=[_('ScanStrategy')], - summary=_('ScanStrategy Update'), - description=_("Get the item corresponding to the user, support fuzzy search based on name." - ), - ) - def update(self, request, pk): - ser = ScanCreateSerializer(data=request.data, partial=True) - try: - if ser.is_valid(True): - pass - except ValidationError as e: - return R.failure(data=e.detail) - if ser.validated_data.get('content', None): - ser.validated_data['content'] = ','.join( - [str(i) for i in ser.validated_data['content']]) - obj = IastStrategyUser.objects.filter(pk=pk).update( - **ser.validated_data) - disable_cache(load_sink_strategy, (), kwargs={"scan_id": pk}) - return R.success(msg=_('update success')) - - @extend_schema_with_envcheck( - tags=[_('ScanStrategy')], - summary=_('ScanStrategy delete'), - description=_("Get the item corresponding to the user, support fuzzy search based on name." - ), - ) - def destory(self, request, pk): - scan = IastStrategyUser.objects.filter(pk=pk).first() - if not scan: - return R.failure(msg='No scan strategy found') - if checkusing(scan): - return R.failure(msg='someproject is using this scan strategy') - try: - IastStrategyUser.objects.filter(pk=pk, ).update(status=-1) - return R.success(msg=_('delete success')) - except Exception as e: - logger.error(e) - return R.failure() - - @extend_schema_with_envcheck( - tags=[_('ScanStrategy')], - summary=_('ScanStrategy get'), - description=_("Get the item with pk"), - ) - def retrieve(self, request, pk): - obj = IastStrategyUser.objects.filter(pk=pk).first() - return R.success(data=ScanStrategySerializer(obj).data) - - -class ScanStrategyBatchView(BatchStatusUpdateSerializerView): - status_field = 'status' - model = IastStrategyUser - - @extend_schema_with_envcheck( - request=BatchStatusUpdateSerializerView.serializer, - tags=[_('ScanStrategy')], - summary=_('ScanStrategy batch status'), - description=_("batch update status."), - ) - def post(self, request): - data = self.get_params(request.data) - user = request.user - data['ids'] = filter_using(data['ids'], [user]) - self.update_model(request, data) - return R.success(msg=_('update success')) - - -class ScanStrategyAllView(AllStatusUpdateSerializerView): - status_field = 'status' - model = IastStrategyUser - - @extend_schema_with_envcheck( - request=BatchStatusUpdateSerializerView.serializer, - tags=[_('ScanStrategy')], - summary=_('ScanStrategy all status'), - description=_("all update status."), - ) - def post(self, request): - data = self.get_params(request.data) - self.update_model(request, data) - return R.success(msg=_('update success')) - - def update_model(self, request, validated_data): - ids = self.model.objects.values_list('id', flat=True).all() - filter_ids = filter_using(ids, request.user) - self.model.objects.filter(pk__in=filter_ids, - user__in=[request.user]).update(**{ - self.status_field: - validated_data['status'] - }) - - -def filter_using(ids, users): - after_filter_ids = [] - for obj in IastStrategyUser.objects.filter(pk__in=ids, user__in=[users]).all(): - if checkusing(obj): - continue - after_filter_ids.append(obj.id) - return after_filter_ids - - -def checkusing(scanstrategy): - return IastProject.objects.filter(scan=scanstrategy).exists() diff --git a/test/iast/views/test_scan_strategy.py b/test/iast/views/test_scan_strategy.py deleted file mode 100644 index 82383977a..000000000 --- a/test/iast/views/test_scan_strategy.py +++ /dev/null @@ -1,24 +0,0 @@ -###################################################################### -# @author : bidaya0 (bidaya0@$HOSTNAME) -# @file : scan_strategy -# @created : 星期四 12月 02, 2021 19:57:44 CST -# -# @description : -###################################################################### - - -from rest_framework.test import APITestCase -from dongtai_common.models.user import User - - -class ScanStrategyTestCase(APITestCase): - def setUp(self): - pass - - def test_create(self): - self.user = User.objects.filter(pk=1).first() - assert self.user is not None - self.client.force_authenticate(user=self.user) - response = self.client.get('/api/v1/scan_strategy') - print(response.content) - assert response.status_code == 200 From f9576ece36df9bdfed825cf036b1c300b23b557e Mon Sep 17 00:00:00 2001 From: st1020 Date: Fri, 7 Jul 2023 10:38:33 +0800 Subject: [PATCH 065/161] refactor: remove match statement --- dongtai_common/endpoint/__init__.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/dongtai_common/endpoint/__init__.py b/dongtai_common/endpoint/__init__.py index 953f2bce4..0f7cfa42b 100644 --- a/dongtai_common/endpoint/__init__.py +++ b/dongtai_common/endpoint/__init__.py @@ -135,17 +135,16 @@ def dispatch(self, request, *args, **kwargs): if operate_tag: operate_method = operate_tag[0].lstrip("operate-") - match operate_method: - case "GET": - operate_type = OperateType.GET - case "POST": - operate_type = OperateType.ADD - case "PUT": - operate_type = OperateType.CHANGE - case "DELETE": - operate_type = OperateType.DELETE - case _: - raise ValueError("unknown request method") + if operate_method == "GET": + operate_type = OperateType.GET + elif operate_method == "POST": + operate_type = OperateType.ADD + elif operate_method == "PUT": + operate_type = OperateType.CHANGE + elif operate_method == "DELETE": + operate_type = OperateType.DELETE + else: + raise ValueError("unknown request method") IastLog.objects.create( url=path, From 5d78641fd3d580b3e5540bddd271bbd2b0500dce Mon Sep 17 00:00:00 2001 From: st1020 Date: Fri, 7 Jul 2023 11:52:42 +0800 Subject: [PATCH 066/161] fix: agent status log --- dongtai_engine/tasks.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dongtai_engine/tasks.py b/dongtai_engine/tasks.py index eacaffe2a..d1f7cbc4a 100644 --- a/dongtai_engine/tasks.py +++ b/dongtai_engine/tasks.py @@ -337,12 +337,17 @@ def update_agent_status(): ).select_related("agent__user") for _, vul_list in groupby(vuls, lambda x: x.agent.user_id): vul_list = list(vul_list) + replay_queue = IastReplayQueue.objects.filter( + relation_id__in=list(map(lambda x: x.id, vul_list)), + state__in=(const.PENDING, const.WAITING), + ).all() log_recheck_vul( vul_list[0].agent.user.id, vul_list[0].agent.user.username, - list(map(lambda x: x.id, vul_list)), + list(map(lambda x: x.relation_id, replay_queue)), "验证失败", ) + replay_queue.update(state=const.DISCARD) logger.info("update offline agent: %s", stop_agent_ids) logger.info('检测引擎状态更新成功') after_agent_status_update() From c4154d3d3a231fafd9814d18511d39d9d8196711 Mon Sep 17 00:00:00 2001 From: st1020 Date: Fri, 7 Jul 2023 12:12:33 +0800 Subject: [PATCH 067/161] feat: remove log api --- dongtai_web/urls.py | 8 --- dongtai_web/views/log_clear.py | 34 ------------ dongtai_web/views/log_delete.py | 35 ------------- dongtai_web/views/log_export.py | 70 ------------------------- dongtai_web/views/logs.py | 93 --------------------------------- 5 files changed, 240 deletions(-) delete mode 100644 dongtai_web/views/log_clear.py delete mode 100644 dongtai_web/views/log_delete.py delete mode 100644 dongtai_web/views/log_export.py delete mode 100644 dongtai_web/views/logs.py diff --git a/dongtai_web/urls.py b/dongtai_web/urls.py index 4a0351a7c..d8fdc463b 100644 --- a/dongtai_web/urls.py +++ b/dongtai_web/urls.py @@ -47,10 +47,6 @@ from dongtai_web.views.engine_hook_rules import EngineHookRulesEndPoint from dongtai_web.views.engine_method_pool_detail import MethodPoolDetailProxy from dongtai_web.views.engine_method_pool_search import MethodPoolSearchProxy -from dongtai_web.views.log_clear import LogClear -from dongtai_web.views.log_delete import LogDelete -from dongtai_web.views.log_export import LogExport -from dongtai_web.views.logs import LogsEndpoint from dongtai_web.views.method_graph import MethodGraph from dongtai_web.views.openapi import OpenApiEndpoint from dongtai_web.views.profile import ProfileEndpoint, ProfileBatchGetEndpoint, ProfileBatchModifiedEndpoint @@ -228,10 +224,6 @@ path('profile/batch/get', ProfileBatchGetEndpoint.as_view()), path('profile/batch/modified', ProfileBatchModifiedEndpoint.as_view()), path('system/info', SystemInfo.as_view()), - path('logs', LogsEndpoint.as_view()), - path('log/export', LogExport.as_view()), - path('log/delete', LogDelete.as_view()), - path('log/clear', LogClear.as_view()), path('engine/method_pool/search', MethodPoolSearchProxy.as_view()), path('engine/method_pool/detail', MethodPoolDetailProxy.as_view()), path('engine/method_pool/timerange', MethodPoolTimeRangeProxy.as_view()), diff --git a/dongtai_web/views/log_clear.py b/dongtai_web/views/log_clear.py deleted file mode 100644 index 7b620dd21..000000000 --- a/dongtai_web/views/log_clear.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python -# -*- coding:utf-8 -*- -# author:owefsad -# software: PyCharm -# project: lingzhi-webapi - -from django.contrib.admin.models import LogEntry -from dongtai_common.endpoint import UserEndPoint, R -from django.utils.translation import gettext_lazy as _ -import datetime -from drf_spectacular.utils import extend_schema - - -class LogClear(UserEndPoint): - name = 'api-v1-log-clear' - description = _('Log clear') - - @extend_schema( - deprecated=True, - summary=_('Log clear'), - tags=[_("Logs")], - ) - def get(self, request): - user = request.user - now = datetime.datetime.now() - - if user.is_system_admin(): - LogEntry.objects.filter(action_time__lt=now).delete() - elif user.is_talent_admin(): - users = self.get_auth_users(user) - LogEntry.objects.filter(action_time__lt=now, user__in=users).delete() - else: - return R.failure(status=203, msg=_('no permission')) - return R.success() diff --git a/dongtai_web/views/log_delete.py b/dongtai_web/views/log_delete.py deleted file mode 100644 index 6d72c80b7..000000000 --- a/dongtai_web/views/log_delete.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python -# -*- coding:utf-8 -*- -# author:owefsad -# software: PyCharm -# project: lingzhi-webapi - -from django.contrib.admin.models import LogEntry -from dongtai_common.endpoint import UserEndPoint, R -from django.utils.translation import gettext_lazy as _ -from drf_spectacular.utils import extend_schema - - -class LogDelete(UserEndPoint): - name = 'api-v1-log-delete' - description = _('Log delete') - - @extend_schema( - deprecated=True, - summary=_('Log delete'), - tags=[_("Logs")] - ) - def post(self, request): - ids = request.data.get('ids') - if ids: - ids = [int(id.strip()) for id in ids.split(',')] - user = request.user - if user.is_superuser == 1: - LogEntry.objects.filter(id__in=ids).delete() - return R.success(msg=_('success')) - users = self.get_auth_users(user) - user_ids = list(users.values_list('id', flat=True)) - LogEntry.objects.filter(id__in=ids, user_id__in=user_ids).delete() - return R.success(msg=_('success')) - else: - return R.failure(status=203, msg=_('The data to be deleted should not be empty')) diff --git a/dongtai_web/views/log_export.py b/dongtai_web/views/log_export.py deleted file mode 100644 index 4f1ec732f..000000000 --- a/dongtai_web/views/log_export.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python -# -*- coding:utf-8 -*- -# author:owefsad -# software: PyCharm -# project: lingzhi-webapi -from django.contrib.admin.models import LogEntry -from django.http import HttpResponse -from django.utils.encoding import escape_uri_path -from import_export import resources -from rest_framework.generics import GenericAPIView -from django.utils.translation import gettext_lazy as _ -from dongtai_common.endpoint import UserEndPoint -from dongtai_common.endpoint import R -from drf_spectacular.utils import extend_schema - - -class LogResurce(resources.ModelResource): - def get_export_headers(self): - return [ - u'时间', u'用户', u'操作记录' - ] - - class Meta: - model = LogEntry - fields = ('user', 'action_time', 'change_message') - - -class ExportMixin(object): - @staticmethod - def attachment_response(export_data, filename='download.xls', content_type='application/vnd.ms-excel'): - """ - - https://segmentfault.com/q/1010000009719860 - - https://blog.csdn.net/qq_34309753/article/details/99628474 - - :param export_data: - :param filename: - :param content_type: - :return: - """ - response = HttpResponse(export_data, content_type=content_type) - response['content_type'] = content_type - response['Content-Disposition'] = "attachment; filename*=utf-8''{}".format(escape_uri_path(filename)) - return response - - @extend_schema( - deprecated=True, - summary="日志导出", - tags=[_("Logs")] - ) - def get(self, request): - ids = request.query_params.get('ids') - if ids: - ids = [int(id.strip()) for id in ids.split(',')] - user = request.user - if user.is_system_admin(): - queryset = LogEntry.objects.filter(id__in=ids).filter() - elif user.is_talent_admin(): - auth_users = UserEndPoint.get_auth_users(user) - queryset = LogEntry.objects.filter(id__in=ids, user__in=auth_users).filter() - else: - return R.failure(msg=_('no permission')) - resources = self.resource_class() - export_data = resources.export(queryset, False) - return ExportMixin.attachment_response(getattr(export_data, 'xls'), filename='用户操作日志.xls') - else: - return R.failure(status=202, msg=_('Export failed, error message: Log id should not be empty')) - - -class LogExport(ExportMixin, GenericAPIView): - resource_class = LogResurce diff --git a/dongtai_web/views/logs.py b/dongtai_web/views/logs.py deleted file mode 100644 index 004c6045c..000000000 --- a/dongtai_web/views/logs.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python -# -*- coding:utf-8 -*- -# author:sjh -# software: PyCharm -# project: webapi -import logging -from django.contrib.admin.models import LogEntry -from dongtai_common.endpoint import UserEndPoint, R -from django.utils.translation import gettext_lazy as _ -from django.core.cache import cache -from drf_spectacular.utils import extend_schema - -logger = logging.getLogger('dongtai-webapi') - - -class LogsEndpoint(UserEndPoint): - name = 'api-v1-logs' - description = _('Log list') - - def make_key(self, request): - self.cache_key = f"{request.user.id}_total_logs_id" - self.cache_key_max_id = f"{request.user.id}_max_logs_id" - - def get_query_cache(self): - total = cache.get(self.cache_key) - max_id = cache.get(self.cache_key_max_id) - return total, max_id - - def set_query_cache(self, queryset): - total = queryset.values('id').count() - if total > 0: - max_id = queryset.values_list('id', flat=True).order_by('-id')[0] - else: - max_id = 0 - cache.set(self.cache_key, total, 60 * 60) - cache.set(self.cache_key_max_id, max_id, 60 * 60) - return total, max_id - - def parse_args(self, request): - page = int(request.query_params.get('page', 1)) - page_size = int(request.query_params.get('pageSize', 20)) - page_size = page_size if page_size < 50 else 50 - return page, page_size, request.user - - @extend_schema( - deprecated=True, - summary="获取日志列表", - tags=[_("Logs")] - ) - def get(self, request): - try: - page, page_size, user = self.parse_args(request) - - if user.is_system_admin(): - queryset = LogEntry.objects.all() - elif user.is_talent_admin(): - users = self.get_auth_users(user) - user_ids = list(users.values_list('id', flat=True)) - queryset = LogEntry.objects.filter(user_id__in=user_ids) - else: - queryset = LogEntry.objects.filter(user=user) - # set cache key - self.make_key(request) - if page == 1: - total, max_id = self.set_query_cache(queryset) - else: - total, max_id = self.get_query_cache() - if not total or not max_id: - total, max_id = self.set_query_cache(queryset) - # only read log_id - cur_data = queryset.filter(id__lte=max_id).values_list('id', flat=True).order_by('-id')[(page - 1) * page_size: page * page_size] - cur_ids = [] - for item in cur_data: - cur_ids.append(item) - # read log detail - page_data = LogEntry.objects.filter(id__in=cur_ids).order_by('-id').select_related('content_type', 'user') - data = [] - for item in page_data: - data.append({ - "log_id": item.id, - "user_id": item.user.id, - "username": item.user.username, - "action_time": item.action_time.strftime('%Y-%m-%d %H:%M:%S'), - "content_type": item.content_type.app_labeled_name, - "object_id": item.object_id, - "object_repr": item.object_repr, - "action_flag": item.action_flag, - "change_message": item.change_message, - }) - return R.success(data=data, total=total) - except Exception as e: - logger.error(e, exc_info=True) - return R.success(data=list(), msg=_('failure')) From bf81c1124f2258bded25a069e4e7fe13623aecf7 Mon Sep 17 00:00:00 2001 From: st1020 Date: Fri, 7 Jul 2023 12:35:29 +0800 Subject: [PATCH 068/161] feat: remove recognize_rule api --- dongtai_web/project/__init__.py | 0 dongtai_web/project/recognize_rule.py | 204 -------------------------- dongtai_web/urls.py | 15 +- 3 files changed, 1 insertion(+), 218 deletions(-) delete mode 100644 dongtai_web/project/__init__.py delete mode 100644 dongtai_web/project/recognize_rule.py diff --git a/dongtai_web/project/__init__.py b/dongtai_web/project/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/dongtai_web/project/recognize_rule.py b/dongtai_web/project/recognize_rule.py deleted file mode 100644 index 7dee9a8c3..000000000 --- a/dongtai_web/project/recognize_rule.py +++ /dev/null @@ -1,204 +0,0 @@ -from dongtai_common.utils import const -import logging - -from dongtai_common.endpoint import R -from dongtai_common.utils import const -from dongtai_common.endpoint import UserEndPoint -from django.forms.models import model_to_dict -from django.db.models import Q -from rest_framework import serializers -from rest_framework.serializers import ValidationError -from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer -from rest_framework.viewsets import ViewSet -from django.utils.translation import gettext_lazy as _ -from django.db import models -from dongtai_common.models.recognize_rule import ( - IastRecognizeRule, - RuleTypeChoices, -) - -logger = logging.getLogger('dongtai-webapi') - - -class DeleteTypeChoices(models.IntegerChoices): - ALL = 1 - BATCH = 2 - -class RecognizeRuleListSerializer(serializers.Serializer): - page_size = serializers.IntegerField(default=20, - help_text=_('Number per page')) - page = serializers.IntegerField(default=1, help_text=_('Page index')) - project_id = serializers.IntegerField(help_text=_('Page index'), - required=True) - rule_type = serializers.ChoiceField( - help_text=_('Rule type'), - required=True, - choices=RuleTypeChoices, - ) - - -class RecognizeRuleSerializer(serializers.ModelSerializer): - - class Meta: - model = IastRecognizeRule - fields = ['rule_detail', 'rule_type', 'pk'] - - -class RecognizeRuleCreateSerializer(serializers.ModelSerializer): - project_id = serializers.IntegerField(min_value=1, - max_value=2**31 - 1, - help_text=_('project id'), - required=True) - - class Meta: - model = IastRecognizeRule - fields = ['project_id', 'rule_detail', 'rule_type'] - - -class RecognizeRuleBatchDeleteSerializer(serializers.Serializer): - rule_type = serializers.ChoiceField( - help_text=_('Rule type'), - required=True, - choices=RuleTypeChoices, - ) - delete_type = serializers.ChoiceField( - help_text=_('Delete Type'), - required=True, - choices=DeleteTypeChoices, - ) - project_id = serializers.IntegerField(help_text=_('project id'), - required=True) - delete_ids = serializers.ListField( - child=serializers.IntegerField(required=True, ), - help_text=_('Delete Type'), - required=False, - default=[], - ) - - -class RecognizeRuleViewSet(UserEndPoint, ViewSet): - - permission_classes_by_action = {} - - def get_permissions(self): - try: - return [ - permission() for permission in - self.permission_classes_by_action[self.action] - ] - except KeyError: - return [permission() for permission in self.permission_classes] - - @extend_schema_with_envcheck( - [RecognizeRuleListSerializer], - tags=[_('RecognizeRule')], - summary=_('RecognizeRule List'), - ) - def list(self, request): - ser = RecognizeRuleListSerializer(data=request.GET) - try: - if ser.is_valid(True): - pass - except ValidationError as e: - return R.failure(data=e.detail) - q = Q() - if ser.validated_data['project_id']: - q = q & Q(project_id=ser.validated_data['project_id']) - if ser.validated_data['rule_type']: - q = q & Q(rule_type=ser.validated_data['rule_type']) - queryset = IastRecognizeRule.objects.filter(q).order_by('-updated') - page_summary, page_data = self.get_paginator( - queryset, ser.validated_data['page'], - ser.validated_data['page_size']) - return R.success(data=RecognizeRuleSerializer(page_data, - many=True).data, - page=page_summary) - - @extend_schema_with_envcheck( - request=RecognizeRuleCreateSerializer, - tags=[_('RecognizeRule')], - summary=_('RecognizeRule Create'), - description=_("Create RecognizeRule"), - ) - def create(self, request): - ser = RecognizeRuleCreateSerializer(data=request.data) - try: - if ser.is_valid(True): - pass - except ValidationError as e: - return R.failure(data=e.detail) - try: - obj = IastRecognizeRule.objects.create(**ser.validated_data, ) - return R.success(msg=_('create success'), - data=RecognizeRuleSerializer(obj).data) - except Exception as e: - logger.error(e) - return R.failure() - - @extend_schema_with_envcheck( - request=RecognizeRuleCreateSerializer, - tags=[_('RecognizeRule')], - summary=_('RecognizeRule Update'), - ) - def update(self, request, pk): - ser = RecognizeRuleCreateSerializer(data=request.data) - try: - if ser.is_valid(True): - pass - if not pk > 0: - return R.failure() - except ValidationError as e: - return R.failure(data=e.detail) - obj = IastRecognizeRule.objects.filter(pk=pk).update( - **ser.validated_data) - return R.success(msg=_('update success')) - - @extend_schema_with_envcheck( - request=RecognizeRuleBatchDeleteSerializer, - tags=[_('RecognizeRule')], - summary=_('RecognizeRule delete'), - ) - def destory(self, request): - """ - Example: - - ------------------------- - { - "delete_type":1, - "project_id":1 - } - ------------------------- - { - "delete_type":2, - "project_id":1, - "delete_ids": [1,2,3] - } - ------------------------ - """ - ser = RecognizeRuleBatchDeleteSerializer(data=request.data) - try: - if ser.is_valid(True): - pass - except ValidationError as e: - return R.failure(data=e.detail) - if ser.validated_data['delete_type'] == DeleteTypeChoices.ALL: - IastRecognizeRule.objects.filter( - project_id=ser.validated_data['project_id'], - rule_type=ser.validated_data['rule_type'], - ).delete() - else: - IastRecognizeRule.objects.filter( - project_id=ser.validated_data['project_id'], - pk__in=ser.validated_data['delete_ids'], - rule_type=ser.validated_data['rule_type'], - ).delete() - return R.success(msg=_('delete success')) - - @extend_schema_with_envcheck( - tags=[_('RecognizeRule')], - summary=_('RecognizeRule get'), - description=_("Get the item with pk"), - ) - def retrieve(self, request, pk): - obj = IastRecognizeRule.objects.filter(pk=pk).first() - return R.success(data=RecognizeRuleSerializer(obj).data) diff --git a/dongtai_web/urls.py b/dongtai_web/urls.py index d8fdc463b..4178102d1 100644 --- a/dongtai_web/urls.py +++ b/dongtai_web/urls.py @@ -146,8 +146,7 @@ from dongtai_web.dast.webhook import DastWebhook from dongtai_web.dast.page import DastVulsEndPoint from dongtai_web.dast.manage import DastManageEndPoint -from dongtai_web.views.new_project_query import (NewApiRouteSearch, NewProjectVersionList) -from dongtai_web.project.recognize_rule import RecognizeRuleViewSet +from dongtai_web.views.new_project_query import NewApiRouteSearch, NewProjectVersionList from dongtai_web.enum.hook_rules import HookRuleEnumEndPoint from django.urls import URLResolver, URLPattern @@ -384,18 +383,6 @@ DastManageEndPoint.as_view({ 'get': "get_doc_url", })), - path('project/recognize_rule/', - RecognizeRuleViewSet.as_view({ - 'get': "retrieve", - 'put': "update", - })), - path( - 'project/recognize_rule', - RecognizeRuleViewSet.as_view({ - 'post': "create", - 'get': "list", - 'delete': "destory", - })), path('hook_rule/enum', HookRuleEnumEndPoint.as_view({ 'get': "get_enums", })), From 58980b821a71ebb60b11daf32b12c43bf4b67783 Mon Sep 17 00:00:00 2001 From: st1020 Date: Fri, 7 Jul 2023 14:28:11 +0800 Subject: [PATCH 069/161] feat: remove dast api --- dongtai_web/dast/__init__.py | 0 dongtai_web/dast/manage.py | 78 ----- dongtai_web/dast/mockdata/method_pool.pickle | Bin 12266 -> 0 bytes dongtai_web/dast/page.py | 313 ------------------ dongtai_web/dast/tests.py | 325 ------------------- dongtai_web/dast/webhook.py | 156 --------- dongtai_web/urls.py | 38 --- 7 files changed, 910 deletions(-) delete mode 100644 dongtai_web/dast/__init__.py delete mode 100644 dongtai_web/dast/manage.py delete mode 100644 dongtai_web/dast/mockdata/method_pool.pickle delete mode 100644 dongtai_web/dast/page.py delete mode 100644 dongtai_web/dast/tests.py delete mode 100644 dongtai_web/dast/webhook.py diff --git a/dongtai_web/dast/__init__.py b/dongtai_web/dast/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/dongtai_web/dast/manage.py b/dongtai_web/dast/manage.py deleted file mode 100644 index 4fc6f8c4e..000000000 --- a/dongtai_web/dast/manage.py +++ /dev/null @@ -1,78 +0,0 @@ -import logging -import json - -from django.db.models import Q, F, Count -from django.utils.translation import gettext_lazy as _ -from dongtai_common.endpoint import UserEndPoint, R -from rest_framework.viewsets import ViewSet -from dongtai_common.models.agent import IastAgent -from dongtai_common.models.strategy import IastStrategyModel -from dongtai_common.models.vulnerablity import IastVulnerabilityModel -from django.core.cache import cache -from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer -from dongtai_common.models.dast_integration import IastDastIntegration -from rest_framework import serializers -from rest_framework import viewsets -from rest_framework.serializers import ValidationError -from dongtai_common.models.profile import IastProfile - -logger = logging.getLogger('dongtai-webapi') - - -class DastVulsSettingsArgsSerializer(serializers.Serializer): - strategy_id = serializers.ListField(child=serializers.IntegerField(), - required=True, - help_text=_("strategy_id")) - - validation_status = serializers.BooleanField( - required=True, help_text=_("cross validation status enable")) - - -class DastManageEndPoint(UserEndPoint, viewsets.ViewSet): - - @extend_schema_with_envcheck(request=DastVulsSettingsArgsSerializer, - summary=_('Dast Vul Settings'), - description=_("Dast Vul Settings"), - tags=[_('Dast Vul')]) - def change_validation_settings(self, request): - ser = DastVulsSettingsArgsSerializer(data=request.data) - try: - if ser.is_valid(True): - pass - except ValidationError as e: - return R.failure(data=e.detail) - key = 'dast_validation_settings' - try: - obj, created = IastProfile.objects.update_or_create( - { - 'key': key, - 'value': json.dumps(dict(ser.validated_data)) - }, - key=key) - except Exception as e: - logger.error(e, exc_info=e) - return R.failure(msg=_("Update {} failed").format(key)) - return R.success() - - @extend_schema_with_envcheck(request=DastVulsSettingsArgsSerializer, - summary=_('Dast Vul Settings'), - description=_("Dast Vul Settings"), - tags=[_('Dast Vul')]) - def get_validation_settings(self, request): - key = 'dast_validation_settings' - profile = IastProfile.objects.filter(key=key).values_list( - 'value', flat=True).first() - if profile is None: - return R.failure( - msg=_("Failed to get {} configuration").format(key)) - data = json.loads(profile) - return R.success(data=data) - - @extend_schema_with_envcheck(summary=_('Dast Vul Settings Doc'), - description=_("Dast Vul Settings Doc"), - tags=[_('Dast Vul')]) - def get_doc_url(self, request): - return R.success(data={ - "url": - "https://i0x0fy4ibf.feishu.cn/docx/GGpUdYopaoxD4oxnOlncKKbsnRh" - }) diff --git a/dongtai_web/dast/mockdata/method_pool.pickle b/dongtai_web/dast/mockdata/method_pool.pickle deleted file mode 100644 index 715285382d288ed4475ce473fbbc4b3d2734e15d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12266 zcmds7TW=f36&BJUO_Vf!DiHLEAZ(a+t>xax-K8a^6iHFGMN*Ax9H|IWQ&dgsv|HprmEA*%L z2fc;tbBEDYrtRq1@);HRxZnGhertrbYibu3?*9l)x?`UQ$ZTj%yY1Nh-f9U4hNJH} zj@AEL|A&6H|GuB-ebMj(6yUzG0s{!-0sZ=8|LI?8|2It&Te{!SkY-?L>3xB8-L%j9 zfA4*!tNq?@b=*Xu74$>3_nE2pOZ|WLzC`EP4jS<7^%uS0gr3#!aYhihnPievc!o>M z49jq=l#yAPO?E=d_LD5juQg2zuXz~hOv`usWTx5geYbeP*95fDrfk9RV&Smgi`sl3 zwheOmoQ&xAen$too)b74aNqlK=kRbZ$uS&A#lvqK2CRv_{>gWrt*q2*<<{xR7Q0*P z8fshT>NVY}*simCRIcpqxK2eVD5ad3%C)O7hBLG_2(Wl;oEDF-D(xe_)Oux=TXmsa zI}$4S{j1ZKZkFR#P;FR)Os+2$KLc4HUthP$|!s{XZ6}xKT zo&65lsRnAUYiK)LKB|=sy=Zk*QxT)~KvZj&e*I+YQY~7o`bpJQd9j0woIm7qw`c`K z2D|(=k7{+m;EYrLsH5}8rk3lvYHiy-Jx~m-n6h_sPUm$i=uOn-)A-}7UBXW-HCW$j9v6?V^{lfoFclYhsO|TA|0c$|!7z{c?T7h= z0}ZfTAIOzv`d- z=i8N)7V03zbeL_>K-melozT_`T@AZ|>DZZA(s6@i;IuUq$jM8rBG>f?F~q~J;{|?_ zVeaq`6qAh9@+A4}SrU{%{w!2N;t6{4oPWCU1jQ0%C+3Hm5hJz|bG(>tdRPk_@9OF5 zS15>0GV;=P!{8u*=+|Z(7u#&2i~_TRV^P%p=?MuUj!6`Po&ho6K6Tx+&0urEWI_J$ z$%+uob-Z&1xn%bsQJrZ(peO%zA`|+71pEf=f?o&N>zEo&+%hcUOCInw36FdidyE&_ zf!W3kv7cAIy9FC0;}2jb+bE92jydeoVHOS;vR%;mgdEZ^Ie*v|%}cF2;9TV2^Qc83?l>U?hVMk)zKPJkRl}bVr+#z)U+ulm`wx&oc&qS=A3b zqyvnRO}A1wVNR&l=wur2IgUnrj@$Db!avx-Fiw+QW3oNsnV*gP1Gu56Mo7Enxe+jM z{vg1K1>F(q9KA<0PRydGm`7mYL^X)!iLud?YB zY~6Jr%se`v0d2$sM6zmVCniQjuv}W2-C=Ix-oT0ffBEp)LK66Xcjn&VwyFEOW+Vso4_>kP}GKZ>BqA`9WusU>Mg{zXLMBN`y#Pz?gM zk!s-xf7BldIz6!kGZvHq2HZ~&6AhG7kkrthbt7n7o2%>Oe7m_CgP(OcEu)xbAkW9a zM*Qe-drgif2YrDVSa=hXv)K3aCrN&wylpVjk!xsF|B$d za~~MHhd0+<@+&$14O0fLlZ^Zl3RC0iQ={qkPgE#CK^ZHysOA;9z{3d&B zg;fLzg&VAqm=!{4Fmr9J<;u^mjTdh>SVkc=hc@yq)}kt02B&!*r_)jjOR~r}C6!gM zsBuVAd0ke;CK*ri6buE5l&)$iB_)Vy5h_}UlXWePb+sugio&xz2X#i4(V_03uuyT} zC0*B8Za`0l;p^_z1bbJ8X6B^XJc! z&*)Hj3cxqdZLF{1F2r8@9KYynr~(}(5q@;!2bmKOU9B3BA;PKsW|ed`bk98~Qm&b{ z4;!6F>ZG(x<7c>4GBMK~iW%FQLn^eH^J~*xB|fr7>#-ESaR{ZIe(xKwX~KTPg^CN* zbR&C%>S9RTA9?gp!?G;1g2GDD zSct=}X~Vd;_Q!BQ#KWn7kGrXVFo-+rpH}uU93p9`pmzBVLYpt-X|6rnnv zFz6|ZN-N7)l(f7>C6k}Fs7+$3OhmC{OkK+2sZHJ%-55V4#R&+92?ztv;r|@_9*vnb zaZwV*w7^U9tcg!q|Gh1Ig!Z5YevkM@WHBeQEoX=68+J&+WZ&AXP6M07Q9xN?{2f>J}@QZS&0+i+$0LKA#*C4-tfDL3UKSz z>pnvNqAYQ=0E`)@3Gd?eAV-E9J}-n+Q>Q`*92a(I*p^_l50C+T6VlXHhZ8i<@%=SO z9%(M=FPJeu4JCJz2hsguMDr~%;vRu|D;%kHPQ2zMb{XdrK?F;tqW>SpegEtiUL#55 zgd=gYhCb~CAH&keEBx9_{eDUJ$ky++ReoTF@{SKTAjf6-_@Zra#uXtJ%~nN(o1Qok z7)LjGP=y)XVpXGR&NkAZ$n$4Tl~r94)w2{&Le5mG{V7ENYtL*&i(g3$c%U))W-|Yi zvlyt|?~%7v|TG3(3Ie*Q6hdA!11e3|88^l;mXg8c2f){P1G-?U~u zd2#vV4pvN*ayRcL_d93L#KoSM=B78nxt>SlIkgAUYolLB%^V_+L<<@Di*z?B5N$lZAaKJX6f8k;$ZY$g2XKox{wLe zJA&!-`NoX-VFfO;a!%U?KaE&?m{jEM0(05Dgv|1h_$75%ypG@b-^- z@9WTo*RxYiU&8D3hHoHvL!MBv3Tq+0DYJ?u=&0G0U`?#C&>+b?k3<2Xlm@SFf8U5+ X@HaYebqSShxO)*8xa^VP{XFwOqdrE^ diff --git a/dongtai_web/dast/page.py b/dongtai_web/dast/page.py deleted file mode 100644 index 4ce57a0aa..000000000 --- a/dongtai_web/dast/page.py +++ /dev/null @@ -1,313 +0,0 @@ -import logging -import json - -from django.db.models import Q, F, Count -from django.utils.translation import gettext_lazy as _ -from dongtai_common.endpoint import UserEndPoint, R -from rest_framework.viewsets import ViewSet -from dongtai_common.models.agent import IastAgent -from dongtai_common.models.strategy import IastStrategyModel -from dongtai_common.models.vulnerablity import IastVulnerabilityModel -from django.core.cache import cache -from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer -from dongtai_common.models.dast_integration import ( - IastDastIntegration, - IastDastIntegrationRelation, - DastvulDtMarkRelation, - IastvulDtMarkRelation, -) -from rest_framework import serializers -from rest_framework import viewsets -from rest_framework.serializers import ValidationError -from collections import defaultdict -from django.db.utils import IntegrityError -logger = logging.getLogger('dongtai-webapi') - - -class VulsPageArgsSerializer(serializers.Serializer): - page_size = serializers.IntegerField(default=20, - help_text=_('Number per page')) - page = serializers.IntegerField(default=1, help_text=_('Page index')) - keyword = serializers.CharField(help_text=_('keyword'), required=False) - project_id = serializers.ListField(child=serializers.IntegerField(), - required=False) - project_version_id = serializers.IntegerField(required=False) - bind_project_id = serializers.IntegerField(required=False) - vul_level_id = serializers.ListField(child=serializers.IntegerField(), - required=False) - vul_type = serializers.ListField(child=serializers.CharField(), - required=False) - order_by_field = serializers.ChoiceField(['create_time', 'vul_level'], - default='create_time') - order_by_order = serializers.ChoiceField(['desc', 'asc'], default='desc') - - -class VulsSummaryArgsSerializer(VulsPageArgsSerializer): - - class Meta: - fields = ['bind_project_id', 'project_id', 'project_version_id'] - - -class VulsDeleteArgsSerializer(serializers.Serializer): - vul_id = serializers.ListField(child=serializers.IntegerField(), - required=True) - - -class VulsResArgsSerializer(serializers.ModelSerializer): - project_name = serializers.CharField(source='project.name') - project_version_name = serializers.CharField( - source='project_version.version_name') - vul_level_name = serializers.CharField(source='vul_level.name') - - class Meta: - model = IastDastIntegration - fields = [ - 'id', - 'vul_name', - 'detail', - 'vul_level', - 'payload', - 'target', - 'vul_type', - 'dast_tag', - 'request_messages', - 'urls', - 'create_time', - 'latest_time', - 'project', - 'project_id', - 'project_version', - 'project_version_id', - 'project_name', - 'project_version_name', - 'vul_level_name', - 'vul_level_id', - ] - - -class IastVulRelationArgsSerializer(serializers.Serializer): - page_size = serializers.IntegerField(default=20, - help_text=_('Number per page')) - page = serializers.IntegerField(default=1, help_text=_('Page index')) - pk = serializers.IntegerField(default=20, help_text=_('iast_vul id')) - is_relatived = serializers.BooleanField(required=False, default=None) - vul_type = serializers.ListField(child=serializers.CharField(), - required=False, - default=[]) - - -class IastVulRelationDeleteArgsSerializer(serializers.Serializer): - iastvul_id = serializers.IntegerField(min_value=1, - max_value=2**31 - 1, - help_text=_('iast_vul id'), - required=True) - dastvul_id = serializers.IntegerField(min_value=1, - max_value=2**31 - 1, - help_text=_('dast_vul id'), - required=True) - - -class DastVulsEndPoint(UserEndPoint, viewsets.ViewSet): - - @extend_schema_with_envcheck(request=VulsSummaryArgsSerializer, - summary=_('Dast Vul Summary'), - description=_("Dast Vul Summary"), - tags=[_('Dast Vul')]) - def summary(self, request): - ser = VulsSummaryArgsSerializer(data=request.data) - try: - if ser.is_valid(True): - pass - except ValidationError as e: - return R.failure(data=e.detail) - department = request.user.get_relative_department() - q = Q(project__department__in=department) - if 'bind_project_id' in ser.validated_data: - q = q & Q(project_id=ser.validated_data['bind_project_id']) - if 'project_version_id' in ser.validated_data: - q = q & Q( - project_version_id=ser.validated_data['project_version_id']) - vul_level_info = IastDastIntegration.objects.filter(q).values( - 'vul_level_id').annotate( - vul_level__name=F("vul_level__name_value"), - total=Count('vul_level_id')).all() - vul_type_info = IastDastIntegration.objects.filter(q).values( - 'vul_type').annotate(total=Count('vul_type')).all() - project_info = IastDastIntegration.objects.filter(q).values( - 'project__name', - 'project_id').annotate(total=Count('project_id')).all() - return R.success( - data={ - "vul_level": list(vul_level_info), - "vul_type": list(vul_type_info), - "project_info": list(project_info), - }) - - @extend_schema_with_envcheck(request=VulsPageArgsSerializer, - summary=_('Dast Vul list'), - description=_("Dast Vul list"), - tags=[_('Dast Vul')]) - def page(self, request): - ser = VulsPageArgsSerializer(data=request.data) - try: - if ser.is_valid(True): - pass - except ValidationError as e: - return R.failure(data=e.detail) - department = request.user.get_relative_department() - q = Q(project__department__in=department) - if 'vul_level_id' in ser.validated_data: - q = q & Q(vul_level_id__in=ser.validated_data['vul_level_id']) - if 'bind_project_id' in ser.validated_data: - q = q & Q(project_id=ser.validated_data['bind_project_id']) - if 'vul_type' in ser.validated_data: - q = q & Q(vul_type__in=ser.validated_data['vul_type']) - if 'project_id' in ser.validated_data: - q = q & Q(project_id__in=ser.validated_data['project_id']) - if 'project_version_id' in ser.validated_data: - q = q & Q( - project_version_id=ser.validated_data['project_version_id']) - if 'keyword' in ser.validated_data: - q = q & Q(vul_name__contains=ser.validated_data['keyword']) - if ser.validated_data['order_by_order'] == 'desc': - order_by_func = F(ser.validated_data['order_by_field']).desc() - else: - order_by_func = F(ser.validated_data['order_by_field']).asc() - page_summary, dastvuls = self.get_paginator( - IastDastIntegration.objects.filter(q).order_by( - order_by_func).all(), ser.validated_data['page'], - ser.validated_data['page_size']) - return R.success(data=VulsResArgsSerializer(dastvuls, many=True).data, - page=page_summary) - - @extend_schema_with_envcheck(summary=_('Dast Vul detail'), - description=_("Dast Vul detail"), - tags=[_('Dast Vul')]) - def single(self, request, pk): - department = request.user.get_relative_department() - q = Q(project__department__in=department) & Q(pk=pk) - dastvul = IastDastIntegration.objects.filter(q).first() - return R.success(data=VulsResArgsSerializer(dastvul).data) - - @extend_schema_with_envcheck([VulsDeleteArgsSerializer], - summary=_('Dast Vul delete'), - description=_("Dast Vul delete"), - tags=[_('Dast Vul')]) - def delete(self, request): - ser = VulsDeleteArgsSerializer(data=request.GET) - try: - if ser.is_valid(True): - pass - except ValidationError as e: - return R.failure(data=e.detail) - department = request.user.get_relative_department() - q = Q(project__department__in=department) & Q( - pk__in=ser.validated_data['vul_id']) - dastvul = IastDastIntegration.objects.filter(q).values().first() - return R.success(data=dastvul) - - @extend_schema_with_envcheck(request=IastVulRelationDeleteArgsSerializer, - summary=_('Dast Vul Relation delete'), - description=_("Dast Vul Relation delete"), - tags=[_('Dast Vul')]) - def delete_relation(self, request): - ser = IastVulRelationDeleteArgsSerializer(data=request.data) - try: - if ser.is_valid(True): - pass - except ValidationError as e: - return R.failure(data=e.detail) - IastDastIntegrationRelation.objects.filter( - iastvul_id=ser.validated_data['iastvul_id'], - dastvul_id=ser.validated_data['dastvul_id'], - ).delete() - return R.success() - - @extend_schema_with_envcheck(request=IastVulRelationDeleteArgsSerializer, - summary=_('Dast Vul Relation create'), - description=_("Dast Vul Relation create"), - tags=[_('Dast Vul')]) - def create_relation(self, request): - ser = IastVulRelationDeleteArgsSerializer(data=request.data) - try: - if ser.is_valid(True): - pass - except ValidationError as e: - return R.failure(data=e.detail) - try: - IastDastIntegrationRelation.objects.create( - dt_mark="manully", - iastvul_id=ser.validated_data['iastvul_id'], - dastvul_id=ser.validated_data['dastvul_id'], - ) - except IntegrityError as e: - logger.debug("confilct exist ,skip") - return R.success() - - @extend_schema_with_envcheck(request=IastVulRelationArgsSerializer, - summary=_('Dast Vul Relation vul'), - description=_("Dast Vul Relation vul"), - tags=[_('Dast Vul')]) - def get_relative_with_dast_vul(self, request): - ser = IastVulRelationArgsSerializer(data=request.data) - try: - if ser.is_valid(True): - pass - except ValidationError as e: - return R.failure(data=e.detail) - q = Q() - if ser.validated_data['is_relatived'] is True: - dt_marks = list( - IastvulDtMarkRelation.objects.filter( - iastvul_id=ser.validated_data['pk']).values_list( - 'dt_mark', flat=True)) + ['manully'] - relatived_dastvul_ids = IastDastIntegrationRelation.objects.filter( - iastvul_id=ser.validated_data['pk']).values( - 'dastvul_id').all() - dastvul_ids = DastvulDtMarkRelation.objects.filter( - dt_mark__in=dt_marks).values('dastvul_id').all() - q = q & Q(pk__in=dastvul_ids) & Q(pk__in=relatived_dastvul_ids) - elif ser.validated_data['is_relatived'] is False: - dt_marks = list( - IastvulDtMarkRelation.objects.filter( - iastvul_id=ser.validated_data['pk']).values_list( - 'dt_mark', flat=True)) + ['manully'] - relatived_dastvul_ids = IastDastIntegrationRelation.objects.filter( - iastvul_id=ser.validated_data['pk']).values( - 'dastvul_id').all() - dastvul_ids = DastvulDtMarkRelation.objects.filter( - dt_mark__in=dt_marks).values('dastvul_id').all() - q = q & Q(pk__in=dastvul_ids) & ~Q(pk__in=relatived_dastvul_ids) - else: - dt_marks = IastvulDtMarkRelation.objects.filter( - iastvul_id=ser.validated_data['pk']).values( - 'dt_mark').all() - dastvul_ids = DastvulDtMarkRelation.objects.filter( - dt_mark__in=dt_marks).values('dastvul_id').all() - q = q & Q(pk__in=dastvul_ids) - if ser.validated_data['vul_type']: - q = q & Q(vul_type__in=ser.validated_data['vul_type']) - page_summary, dastvuls = self.get_paginator( - IastDastIntegration.objects.filter(q).order_by('-pk').all(), - ser.validated_data['page'], ser.validated_data['page_size']) - data = VulsResArgsSerializer(dastvuls, many=True).data - relative_dict = defaultdict( - lambda: False, { - k: True - for k in IastDastIntegrationRelation.objects.filter( - iastvul_id=ser.validated_data['pk'], - dastvul_id__in=[i['id'] for i in data]).values_list( - 'dastvul_id', flat=True).distinct().all() - }) - for item in data: - item['is_relatived'] = relative_dict[item['id']] - return R.success(data=data, - page=page_summary) - - @extend_schema_with_envcheck(summary=_('Dast Vul type'), - description=_("Dast Vul type"), - tags=[_('Dast Vul')]) - def get_vul_type(self, request): - vul_type = IastDastIntegration.objects.values_list( - 'vul_type', flat=True).distinct().all() - return R.success(data=list(vul_type)) diff --git a/dongtai_web/dast/tests.py b/dongtai_web/dast/tests.py deleted file mode 100644 index ae5fc6709..000000000 --- a/dongtai_web/dast/tests.py +++ /dev/null @@ -1,325 +0,0 @@ -from rest_framework.test import APITestCase -from test.apiserver.test_agent_base import AgentTestCase -import json -from dongtai_conf.settings import BASE_DIR -import os -import uuid - -MOCK_DATA_PATH = os.path.join(BASE_DIR, 'dongtai_web/dast/mockdata') - -data1 = { - "dt_mark": ["e622e6675e7842f7b09e4c1a7b2d8b4f"], - "vul_name": - "/vulns/002-file-read.jsp 路径穿越", - "detail": - "test", - "vul_level": - "HIGH", - "urls": ["/vulns/002-file-read.jsp", "/vulns/002-file-read.jsp"], - "payload": - "string", - "create_time": - 2147483647, - "vul_type": - "路径穿越", - "request_messages": [{ - "request": - "GET /vulns/002-file-read.jsp?file=..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2Fetc%2Fpasswd HTTP/1.1\r\nHost: 192.168.0.64:8080\r\nUser-Agent: Xray_Test\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8\r\nAccept-Language: zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2\r\nCookie: JSESSIONID=86F07EDEF41475ADB61064DE440DD395; DTCsrfToken=UzfLT62TGKeJKgNxfNcyoOjEVh124WV3Fl8arbOGzUZDllACTaBOWlin6cRImHt7\r\nDt-Dast: Xray\r\nReferer: http://192.168.0.64:8080/vulns/002-file-read.jsp\r\nUpgrade-Insecure-Requests: 1\r\nXray: x\r\nAccept-Encoding: gzip\r\n\r\n", - "response": - "HTTP/1.1 200 \r\nContent-Length: 5099\r\nContent-Type: text/html;charset=UTF-8\r\nDate: Sat, 18 Mar 2023 07:49:52 GMT\r\nDongtai: v1.9.0-beta1\r\nDt-Request-Id: 23.0794e3c3f73c4c14880b4896ce63b9ec\r\nSet-Cookie: JSESSIONID=7177BD5F10CECCED16BF3745D2CDF6B5; Path=/vulns; HttpOnly\r\n\r\n\n\n\n\n\n\n\n \n 002 任意文件下载/读取漏洞\n\n\n

002 - 任意文件下载/读取漏洞（路径拼接）

\n

正常调用:

\n

curl 'http://192.168.0.64:8080/vulns/002-file-read.jsp?file=example.pdf\n'

\n\n

不正常调用:

\n

curl 'http://192.168.0.64:8080/vulns/002-file-read.jsp?file=../../../../../../../../../../../../../../../etc/passwd\n'

\n\n

不正常调用:

\n

curl 'http://192.168.0.64:8080/vulns/002-file-read.jsp?file=../../../conf/tomcat-users.xml\n'

\n\n
\n

读取内容

\n
root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:/sbin/nologin\ndaemon:x:2:2:daemon:/sbin:/sbin/nologin\nadm:x:3:4:adm:/var/adm:/sbin/nologin\nlp:x:4:7:lp:/var/spool/lpd:/sbin/nologin\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:/sbin/nologin\noperator:x:11:0:operator:/root:/sbin/nologin\ngames:x:12:100:games:/usr/games:/sbin/nologin\nftp:x:14:50:FTP User:/var/ftp:/sbin/nologin\nnobody:x:99:99:Nobody:/:/sbin/nologin\nsystemd-network:x:192:192:systemd Network Management:/:/sbin/nologin\ndbus:x:81:81:System message bus:/:/sbin/nologin\npolkitd:x:999:997:User for polkitd:/:/sbin/nologin\nmysql:x:27:27:MySQL Server:\n"
-    }, {
-        "request":
-        "GET /vulns/002-file-read.jsp?file=..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2Fetc%2Fpasswd HTTP/1.1\r\nHost: 192.168.0.64:8080\r\nUser-Agent: Xray_Test\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8\r\nAccept-Language: zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2\r\nCookie: JSESSIONID=86F07EDEF41475ADB61064DE440DD395; DTCsrfToken=UzfLT62TGKeJKgNxfNcyoOjEVh124WV3Fl8arbOGzUZDllACTaBOWlin6cRImHt7\r\nDt-Dast: Xray\r\nReferer: http://192.168.0.64:8080/vulns/002-file-read.jsp\r\nUpgrade-Insecure-Requests: 1\r\nXray: x\r\nAccept-Encoding: gzip\r\n\r\n",
-        "response":
-        "HTTP/1.1 200 \r\nContent-Length: 5099\r\nContent-Type: text/html;charset=UTF-8\r\nDate: Sat, 18 Mar 2023 07:49:52 GMT\r\nDongtai: v1.9.0-beta1\r\nDt-Request-Id: 23.0794e3c3f73c4c14880b4896ce63b9ec\r\nSet-Cookie: JSESSIONID=7177BD5F10CECCED16BF3745D2CDF6B5; Path=/vulns; HttpOnly\r\n\r\n\n\n\n\n\n\n\n    \n    002 任意文件下载/读取漏洞\n\n\n
002 - 任意文件下载/读取漏洞（路径拼接）
\n
正常调用: 
\n
curl 'http://192.168.0.64:8080/vulns/002-file-read.jsp?file=example.pdf\n'
\n\n
不正常调用: 
\n
curl 'http://192.168.0.64:8080/vulns/002-file-read.jsp?file=../../../../../../../../../../../../../../../etc/passwd\n'
\n\n
不正常调用: 
\n
curl 'http://192.168.0.64:8080/vulns/002-file-read.jsp?file=../../../conf/tomcat-users.xml\n'
\n\n
\n
读取内容
\n
root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:/sbin/nologin\ndaemon:x:2:2:daemon:/sbin:/sbin/nologin\nadm:x:3:4:adm:/var/adm:/sbin/nologin\nlp:x:4:7:lp:/var/spool/lpd:/sbin/nologin\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:/sbin/nologin\noperator:x:11:0:operator:/root:/sbin/nologin\ngames:x:12:100:games:/usr/games:/sbin/nologin\nftp:x:14:50:FTP User:/var/ftp:/sbin/nologin\nnobody:x:99:99:Nobody:/:/sbin/nologin\nsystemd-network:x:192:192:systemd Network Management:/:/sbin/nologin\ndbus:x:81:81:System message bus:/:/sbin/nologin\npolkitd:x:999:997:User for polkitd:/:/sbin/nologin\nmysql:x:27:27:MySQL Server:/var/lib/mysql:/bin/false\n\n"
-    }],
-    "dt_uuid_id": ["213123123122312312312313"],
-    "dongtai_vul_type": ["path-traversal"],
-    "dast_tag":
-    "test",
-    "agent_id": ["1"],
-    "target":
-    "http://192.168.0.64:8080/"
-}
-
-data2 = {
-    "dt_mark": ["e622e6675e7842f7b09e4c1a7b2d8b4f"],
-    "vul_name":
-    "/vulns/002-file-read.jsp 路径穿越",
-    "detail":
-    "test",
-    "vul_level":
-    "HIGH",
-    "urls": [
-        "http://192.168.0.64:8080/vulns/002-file-read.jsp",
-        1,
-    ],
-    "payload":
-    111,
-    "create_time":
-    2147483647,
-    "vul_type":
-    "路径穿越",
-    "request_messages": [{
-        "request":
-        "GET /vulns/002-file-read.jsp?file=..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2Fetc%2Fpasswd HTTP/1.1\r\nHost: 192.168.0.64:8080\r\nUser-Agent: Xray_Test\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8\r\nAccept-Language: zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2\r\nCookie: JSESSIONID=86F07EDEF41475ADB61064DE440DD395; DTCsrfToken=UzfLT62TGKeJKgNxfNcyoOjEVh124WV3Fl8arbOGzUZDllACTaBOWlin6cRImHt7\r\nDt-Dast: Xray\r\nReferer: http://192.168.0.64:8080/vulns/002-file-read.jsp\r\nUpgrade-Insecure-Requests: 1\r\nXray: x\r\nAccept-Encoding: gzip\r\n\r\n",
-    }, {
-        "request":
-        "GET /vulns/002-file-read.jsp?file=..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2Fetc%2Fpasswd HTTP/1.1\r\nHost: 192.168.0.64:8080\r\nUser-Agent: Xray_Test\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8\r\nAccept-Language: zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2\r\nCookie: JSESSIONID=86F07EDEF41475ADB61064DE440DD395; DTCsrfToken=UzfLT62TGKeJKgNxfNcyoOjEVh124WV3Fl8arbOGzUZDllACTaBOWlin6cRImHt7\r\nDt-Dast: Xray\r\nReferer: http://192.168.0.64:8080/vulns/002-file-read.jsp\r\nUpgrade-Insecure-Requests: 1\r\nXray: x\r\nAccept-Encoding: gzip\r\n\r\n",
-        "response":
-        "HTTP/1.1 200 \r\nContent-Length: 5099\r\nContent-Type: text/html;charset=UTF-8\r\nDate: Sat, 18 Mar 2023 07:49:52 GMT\r\nDongtai: v1.9.0-beta1\r\nDt-Request-Id: 23.0794e3c3f73c4c14880b4896ce63b9ec\r\nSet-Cookie: JSESSIONID=7177BD5F10CECCED16BF3745D2CDF6B5; Path=/vulns; HttpOnly\r\n\r\n\n\n\n\n\n\n\n    \n    002 任意文件下载/读取漏洞\n\n\n
002 - 任意文件下载/读取漏洞（路径拼接）
\n
正常调用: 
\n
curl 'http://192.168.0.64:8080/vulns/002-file-read.jsp?file=example.pdf\n'
\n\n
不正常调用: 
\n
curl 'http://192.168.0.64:8080/vulns/002-file-read.jsp?file=../../../../../../../../../../../../../../../etc/passwd\n'
\n\n
不正常调用: 
\n
curl 'http://192.168.0.64:8080/vulns/002-file-read.jsp?file=../../../conf/tomcat-users.xml\n'
\n\n
\n
读取内容
\n
root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:/sbin/nologin\ndaemon:x:2:2:daemon:/sbin:/sbin/nologin\nadm:x:3:4:adm:/var/adm:/sbin/nologin\nlp:x:4:7:lp:/var/spool/lpd:/sbin/nologin\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:/sbin/nologin\noperator:x:11:0:operator:/root:/sbin/nologin\ngames:x:12:100:games:/usr/games:/sbin/nologin\nftp:x:14:50:FTP User:/var/ftp:/sbin/nologin\nnobody:x:99:99:Nobody:/:/sbin/nologin\nsystemd-network:x:192:192:systemd Network Management:/:/sbin/nologin\ndbus:x:81:81:System message bus:/:/sbin/nologin\npolkitd:x:999:997:User for polkitd:/:/sbin/nologin\nmysql:x:27:27:MySQL Server:/var/lib/mysql:/bin/false\n\n"
-    }],
-    "dongtai_vul_type": ["string"],
-    "target":
-    "http://192.168.0.64:8080/"
-}
-
-data3 = {
-    "dt_mark": ["e622e6675e7842f7b09e4c1a7b2d8b4f"],
-    "vul_name":
-    "/vulns/002-file-read.jsp 路径穿越",
-    "detail":
-    "test",
-    "vul_level":
-    "HIGH",
-    "urls": [
-        "http://192.168.0.64:8080/vulns/002-file-read.jsp",
-        "http://192.168.0.64:8080/vulns/002-file-read.jsp"
-    ],
-    "payload":
-    "string",
-    "create_time":
-    2147483647,
-    "vul_type":
-    "路径穿越",
-    "request_messages": [{
-        "request":
-        "GET /vulns/002-file-read.jsp?file=..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2Fetc%2Fpasswd HTTP/1.1\r\nHost: 192.168.0.64:8080\r\nUser-Agent: Xray_Test\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8\r\nAccept-Language: zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2\r\nCookie: JSESSIONID=86F07EDEF41475ADB61064DE440DD395; DTCsrfToken=UzfLT62TGKeJKgNxfNcyoOjEVh124WV3Fl8arbOGzUZDllACTaBOWlin6cRImHt7\r\nDt-Dast: Xray\r\nReferer: http://192.168.0.64:8080/vulns/002-file-read.jsp\r\nUpgrade-Insecure-Requests: 1\r\nXray: x\r\nAccept-Encoding: gzip\r\n\r\n",
-        "response":
-        "HTTP/1.1 200 \r\nContent-Length: 5099\r\nContent-Type: text/html;charset=UTF-8\r\nDate: Sat, 18 Mar 2023 07:49:52 GMT\r\nDongtai: v1.9.0-beta1\r\nDt-Request-Id: 23.0794e3c3f73c4c14880b4896ce63b9ec\r\nSet-Cookie: JSESSIONID=7177BD5F10CECCED16BF3745D2CDF6B5; Path=/vulns; HttpOnly\r\n\r\n\n\n\n\n\n\n\n    \n    002 任意文件下载/读取漏洞\n\n\n
002 - 任意文件下载/读取漏洞（路径拼接）
\n
正常调用: 
\n
curl 'http://192.168.0.64:8080/vulns/002-file-read.jsp?file=example.pdf\n'
\n\n
不正常调用: 
\n
curl 'http://192.168.0.64:8080/vulns/002-file-read.jsp?file=../../../../../../../../../../../../../../../etc/passwd\n'
\n\n
不正常调用: 
\n
curl 'http://192.168.0.64:8080/vulns/002-file-read.jsp?file=../../../conf/tomcat-users.xml\n'
\n\n
\n
读取内容
\n
root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:/sbin/nologin\ndaemon:x:2:2:daemon:/sbin:/sbin/nologin\nadm:x:3:4:adm:/var/adm:/sbin/nologin\nlp:x:4:7:lp:/var/spool/lpd:/sbin/nologin\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:/sbin/nologin\noperator:x:11:0:operator:/root:/sbin/nologin\ngames:x:12:100:games:/usr/games:/sbin/nologin\nftp:x:14:50:FTP User:/var/ftp:/sbin/nologin\nnobody:x:99:99:Nobody:/:/sbin/nologin\nsystemd-network:x:192:192:systemd Network Management:/:/sbin/nologin\ndbus:x:81:81:System message bus:/:/sbin/nologin\npolkitd:x:999:997:User for polkitd:/:/sbin/nologin\nmysql:x:27:27:MySQL Server:\n"
-    }, {
-        "request":
-        "GET /vulns/002-file-read.jsp?file=..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2Fetc%2Fpasswd HTTP/1.1\r\nHost: 192.168.0.64:8080\r\nUser-Agent: Xray_Test\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8\r\nAccept-Language: zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2\r\nCookie: JSESSIONID=86F07EDEF41475ADB61064DE440DD395; DTCsrfToken=UzfLT62TGKeJKgNxfNcyoOjEVh124WV3Fl8arbOGzUZDllACTaBOWlin6cRImHt7\r\nDt-Dast: Xray\r\nReferer: http://192.168.0.64:8080/vulns/002-file-read.jsp\r\nUpgrade-Insecure-Requests: 1\r\nXray: x\r\nAccept-Encoding: gzip\r\n\r\n",
-        "response":
-        "HTTP/1.1 200 \r\nContent-Length: 5099\r\nContent-Type: text/html;charset=UTF-8\r\nDate: Sat, 18 Mar 2023 07:49:52 GMT\r\nDongtai: v1.9.0-beta1\r\nDt-Request-Id: 23.0794e3c3f73c4c14880b4896ce63b9ec\r\nSet-Cookie: JSESSIONID=7177BD5F10CECCED16BF3745D2CDF6B5; Path=/vulns; HttpOnly\r\n\r\n\n\n\n\n\n\n\n    \n    002 任意文件下载/读取漏洞\n\n\n
002 - 任意文件下载/读取漏洞（路径拼接）
\n
正常调用: 
\n
curl 'http://192.168.0.64:8080/vulns/002-file-read.jsp?file=example.pdf\n'
\n\n
不正常调用: 
\n
curl 'http://192.168.0.64:8080/vulns/002-file-read.jsp?file=../../../../../../../../../../../../../../../etc/passwd\n'
\n\n
不正常调用: 
\n
curl 'http://192.168.0.64:8080/vulns/002-file-read.jsp?file=../../../conf/tomcat-users.xml\n'
\n\n
\n
读取内容
\n
root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:/sbin/nologin\ndaemon:x:2:2:daemon:/sbin:/sbin/nologin\nadm:x:3:4:adm:/var/adm:/sbin/nologin\nlp:x:4:7:lp:/var/spool/lpd:/sbin/nologin\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:/sbin/nologin\noperator:x:11:0:operator:/root:/sbin/nologin\ngames:x:12:100:games:/usr/games:/sbin/nologin\nftp:x:14:50:FTP User:/var/ftp:/sbin/nologin\nnobody:x:99:99:Nobody:/:/sbin/nologin\nsystemd-network:x:192:192:systemd Network Management:/:/sbin/nologin\ndbus:x:81:81:System message bus:/:/sbin/nologin\npolkitd:x:999:997:User for polkitd:/:/sbin/nologin\nmysql:x:27:27:MySQL Server:/var/lib/mysql:/bin/false\n\n"
-    }],
-    "dt_uuid_id": ["213123123122312312312313"],
-    "dongtai_vul_type": ["string"],
-    "dast_tag":
-    "test",
-    "agent_id": [],
-    "target":
-    "http://192.168.0.64:8080/"
-}
-
-data4 = {
-    "dt_mark": ["e622e6675e7842f7b09e4c1a7b2d8b4f"],
-    "vul_name":
-    "/vulns/002-file-read.jsp 路径穿越",
-    "detail":
-    "",
-    "vul_level":
-    "HIGH",
-    "urls": [
-        "http://192.168.0.64:8080/vulns/002-file-read.jsp",
-        "http://192.168.0.64:8080/vulns/002-file-read.jsp"
-    ],
-    "payload":
-    "",
-    "create_time":
-    2147483647,
-    "vul_type":
-    "路径穿越",
-    "request_messages": [{
-        "request":
-        "GET /vulns/002-file-read.jsp?file=..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2Fetc%2Fpasswd HTTP/1.1\r\nHost: 192.168.0.64:8080\r\nUser-Agent: Xray_Test\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8\r\nAccept-Language: zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2\r\nCookie: JSESSIONID=86F07EDEF41475ADB61064DE440DD395; DTCsrfToken=UzfLT62TGKeJKgNxfNcyoOjEVh124WV3Fl8arbOGzUZDllACTaBOWlin6cRImHt7\r\nDt-Dast: Xray\r\nReferer: http://192.168.0.64:8080/vulns/002-file-read.jsp\r\nUpgrade-Insecure-Requests: 1\r\nXray: x\r\nAccept-Encoding: gzip\r\n\r\n",
-        "response":
-        "HTTP/1.1 200 \r\nContent-Length: 5099\r\nContent-Type: text/html;charset=UTF-8\r\nDate: Sat, 18 Mar 2023 07:49:52 GMT\r\nDongtai: v1.9.0-beta1\r\nDt-Request-Id: 23.0794e3c3f73c4c14880b4896ce63b9ec\r\nSet-Cookie: JSESSIONID=7177BD5F10CECCED16BF3745D2CDF6B5; Path=/vulns; HttpOnly\r\n\r\n\n\n\n\n\n\n\n    \n    002 任意文件下载/读取漏洞\n\n\n
002 - 任意文件下载/读取漏洞（路径拼接）
\n
正常调用: 
\n
curl 'http://192.168.0.64:8080/vulns/002-file-read.jsp?file=example.pdf\n'
\n\n
不正常调用: 
\n
curl 'http://192.168.0.64:8080/vulns/002-file-read.jsp?file=../../../../../../../../../../../../../../../etc/passwd\n'
\n\n
不正常调用: 
\n
curl 'http://192.168.0.64:8080/vulns/002-file-read.jsp?file=../../../conf/tomcat-users.xml\n'
\n\n
\n
读取内容
\n
root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:/sbin/nologin\ndaemon:x:2:2:daemon:/sbin:/sbin/nologin\nadm:x:3:4:adm:/var/adm:/sbin/nologin\nlp:x:4:7:lp:/var/spool/lpd:/sbin/nologin\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:/sbin/nologin\noperator:x:11:0:operator:/root:/sbin/nologin\ngames:x:12:100:games:/usr/games:/sbin/nologin\nftp:x:14:50:FTP User:/var/ftp:/sbin/nologin\nnobody:x:99:99:Nobody:/:/sbin/nologin\nsystemd-network:x:192:192:systemd Network Management:/:/sbin/nologin\ndbus:x:81:81:System message bus:/:/sbin/nologin\npolkitd:x:999:997:User for polkitd:/:/sbin/nologin\nmysql:x:27:27:MySQL Server:\n"
-    }, {
-        "request":
-        "GET /vulns/002-file-read.jsp?file=..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2F..%2Fetc%2Fpasswd HTTP/1.1\r\nHost: 192.168.0.64:8080\r\nUser-Agent: Xray_Test\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8\r\nAccept-Language: zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2\r\nCookie: JSESSIONID=86F07EDEF41475ADB61064DE440DD395; DTCsrfToken=UzfLT62TGKeJKgNxfNcyoOjEVh124WV3Fl8arbOGzUZDllACTaBOWlin6cRImHt7\r\nDt-Dast: Xray\r\nReferer: http://192.168.0.64:8080/vulns/002-file-read.jsp\r\nUpgrade-Insecure-Requests: 1\r\nXray: x\r\nAccept-Encoding: gzip\r\n\r\n",
-        "response":
-        "HTTP/1.1 200 \r\nContent-Length: 5099\r\nContent-Type: text/html;charset=UTF-8\r\nDate: Sat, 18 Mar 2023 07:49:52 GMT\r\nDongtai: v1.9.0-beta1\r\nDt-Request-Id: 23.0794e3c3f73c4c14880b4896ce63b9ec\r\nSet-Cookie: JSESSIONID=7177BD5F10CECCED16BF3745D2CDF6B5; Path=/vulns; HttpOnly\r\n\r\n\n\n\n\n\n\n\n    \n    002 任意文件下载/读取漏洞\n\n\n
002 - 任意文件下载/读取漏洞（路径拼接）
\n
正常调用: 
\n
curl 'http://192.168.0.64:8080/vulns/002-file-read.jsp?file=example.pdf\n'
\n\n
不正常调用: 
\n
curl 'http://192.168.0.64:8080/vulns/002-file-read.jsp?file=../../../../../../../../../../../../../../../etc/passwd\n'
\n\n
不正常调用: 
\n
curl 'http://192.168.0.64:8080/vulns/002-file-read.jsp?file=../../../conf/tomcat-users.xml\n'
\n\n
\n
读取内容
\n
root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:/sbin/nologin\ndaemon:x:2:2:daemon:/sbin:/sbin/nologin\nadm:x:3:4:adm:/var/adm:/sbin/nologin\nlp:x:4:7:lp:/var/spool/lpd:/sbin/nologin\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:/sbin/nologin\noperator:x:11:0:operator:/root:/sbin/nologin\ngames:x:12:100:games:/usr/games:/sbin/nologin\nftp:x:14:50:FTP User:/var/ftp:/sbin/nologin\nnobody:x:99:99:Nobody:/:/sbin/nologin\nsystemd-network:x:192:192:systemd Network Management:/:/sbin/nologin\ndbus:x:81:81:System message bus:/:/sbin/nologin\npolkitd:x:999:997:User for polkitd:/:/sbin/nologin\nmysql:x:27:27:MySQL Server:/var/lib/mysql:/bin/false\n\n"
-    }],
-    "dt_uuid_id": ["213123123122312312312313"],
-    "dongtai_vul_type": ["string"],
-    "dast_tag":
-    "test",
-    "agent_id": ["1"],
-    "target":
-    "http://192.168.0.64:8080/"
-}
-
-
-class DastWebhookTestCase(AgentTestCase):
-
-    def setUp(self):
-        super().setUp()
-        from dongtai_conf.settings import DAST_TOKEN
-        self.client.credentials(
-            HTTP_X_DONGTAI_DAST_VUL_API_AUTHORIZATION=DAST_TOKEN)
-
-    def test_positive_push_201_2(self):
-        new_data = data4.copy()
-        new_data["agent_id"] = [str(self.agent_id)]
-        res = self.client.post('/api/v1/dast_webhook',
-                               json.dumps(new_data),
-                               content_type="application/json")
-        self.assertEqual(res.status_code, 201)
-
-    def test_positive_push_201_3(self):
-        new_data = data4.copy()
-        new_data["agent_id"] = [str(self.agent_id)]
-        res = self.client.post('/api/v1/dast_webhook',
-                               json.dumps(new_data),
-                               content_type="application/json")
-        self.assertEqual(res.status_code, 201)
-
-    def test_nagetive_push_422(self):
-        new_data = data2.copy()
-        new_data["agent_id"] = [str(self.agent_id)]
-        res = self.client.post('/api/v1/dast_webhook',
-                               json.dumps(new_data),
-                               content_type="application/json")
-        self.assertEqual(res.status_code, 422)
-        pass
-
-    def test_nagetive_push_412(self):
-        res = self.client.post('/api/v1/dast_webhook',
-                               json.dumps(data3),
-                               content_type="application/json")
-        self.assertEqual(res.status_code, 412)
-        pass
-
-    def tearDown(self):
-        pass
-
-
-class IastDastIntegrationBindTestCase(AgentTestCase):
-
-    def setUp(self):
-        super().setUp()
-        from dongtai_conf.settings import DAST_TOKEN
-        self.client.credentials(
-            HTTP_X_DONGTAI_DAST_VUL_API_AUTHORIZATION=DAST_TOKEN)
-
-    def test_positive_push_create(self):
-        new_data = data1.copy()
-        new_data["agent_id"] = [str(self.agent_id)]
-        res = self.client.post('/api/v1/dast_webhook',
-                               json.dumps(new_data),
-                               content_type="application/json")
-        self.assertEqual(res.status_code, 201)
-        from dongtai_engine.tasks import search_vul_from_method_pool
-        from dongtai_common.models.agent_method_pool import MethodPool
-        import pickle
-        with open(os.path.join(MOCK_DATA_PATH, 'method_pool.pickle'),
-                  'rb') as fp:
-            method_pool = pickle.load(fp)
-        method_pool.agent_id = self.agent_id
-        method_pool.save()
-        search_vul_from_method_pool(method_pool.pool_sign,
-                                    method_pool.agent_id)
-        from dongtai_common.models.dast_integration import IastDastIntegrationRelation
-        relcount = IastDastIntegrationRelation.objects.filter(
-            iastvul__method_pool_id=method_pool.id).count()
-        self.assertEqual(relcount, 1)
-
-    def test_positive_create_push(self):
-        from dongtai_engine.tasks import search_vul_from_method_pool
-        from dongtai_common.models.agent_method_pool import MethodPool
-        import pickle
-        with open(os.path.join(MOCK_DATA_PATH, 'method_pool.pickle'),
-                  'rb') as fp:
-            method_pool = pickle.load(fp)
-        method_pool.agent_id = self.agent_id
-        method_pool.save()
-        search_vul_from_method_pool(method_pool.pool_sign,
-                                    method_pool.agent_id)
-        new_data = data1.copy()
-        new_data["agent_id"] = [str(self.agent_id)]
-        res = self.client.post('/api/v1/dast_webhook',
-                               json.dumps(new_data),
-                               content_type="application/json")
-        self.assertEqual(res.status_code, 201)
-        from dongtai_common.models.dast_integration import IastDastIntegrationRelation
-        relcount = IastDastIntegrationRelation.objects.filter(
-            iastvul__method_pool_id=method_pool.id).count()
-        self.assertEqual(relcount, 1)
-
-    def test_positive_create_push_distinct(self):
-        from dongtai_engine.tasks import search_vul_from_method_pool
-        from dongtai_common.models.agent_method_pool import MethodPool
-        import pickle
-        with open(os.path.join(MOCK_DATA_PATH, 'method_pool.pickle'),
-                  'rb') as fp:
-            method_pool = pickle.load(fp)
-        method_pool.agent_id = self.agent_id
-        method_pool.save()
-        search_vul_from_method_pool(method_pool.pool_sign,
-                                    method_pool.agent_id)
-        new_data = data1.copy()
-        new_data["agent_id"] = [str(self.agent_id)]
-        res = self.client.post('/api/v1/dast_webhook',
-                               json.dumps(new_data),
-                               content_type="application/json")
-        self.assertEqual(res.status_code, 201)
-        res = self.client.post('/api/v1/dast_webhook',
-                               json.dumps(new_data),
-                               content_type="application/json")
-        self.assertEqual(res.status_code, 201)
-        res = self.client.post('/api/v1/dast_webhook',
-                               json.dumps(new_data),
-                               content_type="application/json")
-        self.assertEqual(res.status_code, 201)
-        res = self.client.post('/api/v1/dast_webhook',
-                               json.dumps(new_data),
-                               content_type="application/json")
-        self.assertEqual(res.status_code, 201)
-        from dongtai_common.models.dast_integration import IastDastIntegrationRelation
-        relcount = IastDastIntegrationRelation.objects.filter(
-            iastvul__method_pool_id=method_pool.id).count()
-        self.assertEqual(relcount, 1)
-
-    def test_positive_push_create_distinct(self):
-        new_data = data1.copy()
-        new_data["agent_id"] = [str(self.agent_id)]
-        res = self.client.post('/api/v1/dast_webhook',
-                               json.dumps(new_data),
-                               content_type="application/json")
-        self.assertEqual(res.status_code, 201)
-        from dongtai_engine.tasks import search_vul_from_method_pool
-        from dongtai_common.models.agent_method_pool import MethodPool
-        import pickle
-        with open(os.path.join(MOCK_DATA_PATH, 'method_pool.pickle'),
-                  'rb') as fp:
-            method_pool = pickle.load(fp)
-        method_pool.agent_id = self.agent_id
-        method_pool.save()
-        search_vul_from_method_pool(method_pool.pool_sign,
-                                    method_pool.agent_id)
-        method_pool.pool_sign = uuid.uuid4().hex
-        method_pool.id = None
-        method_pool.save()
-        search_vul_from_method_pool(method_pool.pool_sign,
-                                    method_pool.agent_id)
-        method_pool.pool_sign = uuid.uuid4().hex
-        method_pool.id = None
-        method_pool.save()
-        search_vul_from_method_pool(method_pool.pool_sign,
-                                    method_pool.agent_id)
-        method_pool.pool_sign = uuid.uuid4().hex
-        method_pool.id = None
-        method_pool.save()
-        search_vul_from_method_pool(method_pool.pool_sign,
-                                    method_pool.agent_id)
-        method_pool.pool_sign = uuid.uuid4().hex
-        method_pool.id = None
-        method_pool.save()
-        search_vul_from_method_pool(method_pool.pool_sign,
-                                    method_pool.agent_id)
-
-        from dongtai_common.models.dast_integration import IastDastIntegrationRelation
-        relcount = IastDastIntegrationRelation.objects.filter(
-            iastvul__method_pool_id=method_pool.id).count()
-        self.assertEqual(relcount, 1)
diff --git a/dongtai_web/dast/webhook.py b/dongtai_web/dast/webhook.py
deleted file mode 100644
index d23688720..000000000
--- a/dongtai_web/dast/webhook.py
+++ /dev/null
@@ -1,156 +0,0 @@
-import logging
-import json
-
-from django.utils.translation import gettext_lazy as _
-from dongtai_common.endpoint import AnonymousAuthEndPoint, R
-from rest_framework.viewsets import ViewSet
-from dongtai_common.models.agent import IastAgent
-from dongtai_common.models.strategy import IastStrategyModel
-from dongtai_common.models.vulnerablity import IastVulnerabilityModel
-from django.core.cache import cache
-from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
-from dongtai_common.models.dast_integration import (
-    IastDastIntegration,
-    IastDastIntegrationRelation,
-    IastvulDtMarkRelation,
-    DastvulDtMarkRelation,
-)
-from rest_framework import serializers
-from rest_framework.serializers import ValidationError
-from dongtai_conf.settings import DAST_TOKEN
-from dongtai_common.models.profile import IastProfile
-from django.db.utils import IntegrityError
-logger = logging.getLogger('dongtai-webapi')
-
-
-class DastIntegrationRequestMessagesSerializer(serializers.Serializer):
-    request = serializers.CharField()
-    response = serializers.CharField()
-
-
-class DastIntegrationSerializer(serializers.Serializer):
-    dt_uuid_id = serializers.ListField(child=serializers.CharField(),
-                                       required=True)
-    dongtai_vul_type = serializers.ListField(child=serializers.CharField(),
-                                             required=True)
-    agent_id = serializers.ListField(child=serializers.CharField(),
-                                     required=True)
-    request_messages = serializers.ListField(
-        child=DastIntegrationRequestMessagesSerializer(), required=True)
-    vul_level = serializers.ChoiceField(['HIGH', 'MEDIUM', 'LOW', 'NOTE'],
-                                        required=True)
-    urls = serializers.ListField(child=serializers.CharField(), required=True)
-    dt_mark = serializers.ListField(child=serializers.CharField(), required=True)
-    detail = serializers.CharField(required=True, allow_blank=True)
-    payload = serializers.CharField(required=True, allow_blank=True)
-    dast_tag = serializers.CharField(required=True)
-    target = serializers.CharField(required=True, allow_blank=True)
-    vul_name = serializers.CharField(required=True)
-    create_time = serializers.IntegerField(required=True)
-    vul_type = serializers.CharField(required=True)
-
-
-VUL_LEVEL_DICT = {
-    "HIGH": 1,
-    "MEDIUM": 2,
-    "LOW": 3,
-    "NOTE": 5,
-}
-
-
-class DastWebhook(AnonymousAuthEndPoint):
-    name = "api-v1-dast-webhook"
-    description = _("Dast Webhook")
-
-    @extend_schema_with_envcheck(request=DastIntegrationSerializer,
-                                 summary=_('Dast Webhook push vul'),
-                                 description=_("Dast Webhook push vul"),
-                                 tags=[_('Dast Webhook')])
-    def post(self, request):
-        used_token = request.headers.get(
-            'X-Dongtai-Dast-Vul-Api-Authorization', '')
-        if used_token != DAST_TOKEN:
-            return R.failure(msg="Authorization check failed", status_code=401)
-        ser = DastIntegrationSerializer(data=request.data)
-        try:
-            if ser.is_valid(True):
-                pass
-        except ValidationError as e:
-            logger.debug(request.data)
-            logger.info(e.detail)
-            return R.failure(data=e.detail, status_code=422)
-        project_info_set = list(
-            IastAgent.objects.filter(
-                pk__in=(int(i)
-                        for i in ser.validated_data['agent_id'])).values_list(
-                            'bind_project_id',
-                            'project_version_id').distinct(), )
-        dast_list = []
-        vul_level_id = VUL_LEVEL_DICT[ser.validated_data['vul_level']]
-        dt_marks = ser.validated_data['dt_mark']
-        for field in ['dt_uuid_id', 'agent_id', 'vul_level', 'dt_mark']:
-            del ser.validated_data[field]
-        for project_id, project_version_id in project_info_set:
-            dastintegration = IastDastIntegration.objects.filter(
-                project_id=project_id,
-                project_version_id=project_version_id,
-                vul_type=ser.validated_data['vul_type'],
-                target=ser.validated_data['target'],
-            ).first()
-            if dastintegration:
-                logger.debug("dast vul exist, skip")
-            else:
-                dastintegration = IastDastIntegration(
-                    project_id=project_id,
-                    project_version_id=project_version_id,
-                    vul_level_id=vul_level_id,
-                    **ser.validated_data)
-                dastintegration.save()
-            dast_list.append(dastintegration)
-        dastvuldtmarkrel = []
-        for mark in dt_marks:
-            for vul in dast_list:
-                dastvuldtmarkrel.append(
-                    DastvulDtMarkRelation(dt_mark=mark, dastvul=vul))
-            key = 'dast_validation_settings'
-            profile = IastProfile.objects.filter(key=key).values_list(
-                'value', flat=True).first()
-            data = json.loads(profile) if profile else {}
-            if data:
-                match_vul = IastvulDtMarkRelation.objects.filter(
-                    iastvul__uri__in=ser.validated_data['urls'],
-                    iastvul__strategy__vul_type__in=ser.validated_data['dongtai_vul_type'],
-                    iastvul__strategy_id__in=data['strategy_id'],
-                    dt_mark=mark
-                ).values_list('iastvul_id', flat=True)
-            else:
-                match_vul = IastvulDtMarkRelation.objects.filter(
-                    iastvul__uri__in=ser.validated_data['urls'],
-                    iastvul__strategy__vul_type__in=ser.validated_data['dongtai_vul_type'],
-                    dt_mark=mark
-                ).values_list('iastvul_id', flat=True)
-            create_rels = []
-            for iastvul in match_vul:
-                for dastvul in dast_list:
-                    rel = IastDastIntegrationRelation(iastvul_id=iastvul,
-                                                      dastvul=dastvul,
-                                                      dt_mark=mark)
-                    logger.debug(
-                        "create vul_relation iast_vul %s dastvul %s",
-                        iastvul,
-                        dastvul.id,
-                    )
-                    create_rels.append(rel)
-            logger.debug(
-                "create vul_relation count %s with mark %s",
-                len(create_rels),
-                mark,
-            )
-            rels_created = IastDastIntegrationRelation.objects.bulk_create(
-                create_rels, ignore_conflicts=True)
-        mark_created = DastvulDtMarkRelation.objects.bulk_create(
-            dastvuldtmarkrel, ignore_conflicts=True)
-        if dast_list:
-            return R.success(status_code=201)
-        logger.debug(request.data)
-        return R.failure(status_code=412)
diff --git a/dongtai_web/urls.py b/dongtai_web/urls.py
index 4178102d1..c12b37b7c 100644
--- a/dongtai_web/urls.py
+++ b/dongtai_web/urls.py
@@ -143,9 +143,6 @@
 from dongtai_web.vul_log.vul_log_view import VulLogViewSet
 from dongtai_web.vul_recheck_payload.vul_recheck_payload import VulReCheckPayloadViewSet
 from dongtai_web.header_vul.base import HeaderVulViewSet
-from dongtai_web.dast.webhook import DastWebhook
-from dongtai_web.dast.page import DastVulsEndPoint
-from dongtai_web.dast.manage import DastManageEndPoint
 from dongtai_web.views.new_project_query import NewApiRouteSearch, NewProjectVersionList
 from dongtai_web.enum.hook_rules import HookRuleEnumEndPoint
 from django.urls import URLResolver, URLPattern
@@ -348,41 +345,6 @@
     path('header_vul/', HeaderVulViewSet.as_view({
         'delete': "delete",
     })),
-    path('dast_webhook', DastWebhook.as_view()),
-    path('dastvul/', DastVulsEndPoint.as_view({
-        'get': "single",
-    })),
-    path('dastvul',
-         DastVulsEndPoint.as_view({
-             'post': "page",
-             'delete': "delete",
-         })),
-    path(
-        'dastvul/relation',
-        DastVulsEndPoint.as_view({
-            'delete': "delete_relation",
-            'post': "create_relation",
-        })),
-    path('dastvul/relationlist',
-         DastVulsEndPoint.as_view({
-             'post': "get_relative_with_dast_vul",
-         })),
-    path('dastvul/summary', DastVulsEndPoint.as_view({
-        'post': "summary",
-    })),
-    path('dastvul/vultype', DastVulsEndPoint.as_view({
-        'get': "get_vul_type",
-    })),
-    path(
-        'dastvul/settings',
-        DastManageEndPoint.as_view({
-            'post': "change_validation_settings",
-            'get': "get_validation_settings",
-        })),
-    path('dastvul/settings/doc',
-         DastManageEndPoint.as_view({
-             'get': "get_doc_url",
-         })),
     path('hook_rule/enum', HookRuleEnumEndPoint.as_view({
         'get': "get_enums",
     })),

From 774dcaa021c36e91c6051e227d907fe6b5f253b4 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Fri, 7 Jul 2023 14:53:38 +0800
Subject: [PATCH 070/161] feat: remove api_route api

---
 dongtai_web/urls.py                           |  10 +-
 dongtai_web/views/api_route_cover_rate.py     |  87 -----
 .../views/api_route_related_request.py        |  81 ----
 dongtai_web/views/api_route_search.py         | 348 ------------------
 dongtai_web/views/new_project_query.py        |  74 +---
 5 files changed, 6 insertions(+), 594 deletions(-)
 delete mode 100644 dongtai_web/views/api_route_cover_rate.py
 delete mode 100644 dongtai_web/views/api_route_related_request.py
 delete mode 100644 dongtai_web/views/api_route_search.py

diff --git a/dongtai_web/urls.py b/dongtai_web/urls.py
index c12b37b7c..b017a94e7 100644
--- a/dongtai_web/urls.py
+++ b/dongtai_web/urls.py
@@ -103,10 +103,6 @@
 from dongtai_web.views.version_update import MethodPoolVersionUpdate
 from dongtai_web.views.demo import Demo
 from static.i18n.views.setlang import LanguageSetting
-from dongtai_web.views.api_route_search import ApiRouteSearch
-from dongtai_web.views.api_route_related_request import ApiRouteRelationRequest
-from dongtai_web.views.api_route_cover_rate import ApiRouteCoverRate
-
 from dongtai_web.views.program_language import ProgrammingLanguageList
 from dongtai_web.views.filereplace import FileReplace
 from dongtai_web.views.messages_list import MessagesEndpoint
@@ -143,7 +139,7 @@
 from dongtai_web.vul_log.vul_log_view import VulLogViewSet
 from dongtai_web.vul_recheck_payload.vul_recheck_payload import VulReCheckPayloadViewSet
 from dongtai_web.header_vul.base import HeaderVulViewSet
-from dongtai_web.views.new_project_query import NewApiRouteSearch, NewProjectVersionList
+from dongtai_web.views.new_project_query import NewProjectVersionList
 from dongtai_web.enum.hook_rules import HookRuleEnumEndPoint
 from django.urls import URLResolver, URLPattern
 
@@ -240,9 +236,6 @@
     path('version_update/K23DiutPrwpoqAddqNbHUk',
          MethodPoolVersionUpdate.as_view()),
     path('i18n/setlang', LanguageSetting.as_view()),
-    path('api_route/search', ApiRouteSearch.as_view()),
-    path('api_route/relationrequest', ApiRouteRelationRequest.as_view()),
-    path('api_route/cover_rate', ApiRouteCoverRate.as_view()),
     path('program_language', ProgrammingLanguageList.as_view()),
     path('filereplace/', FileReplace.as_view()),
     path('message/list', MessagesEndpoint.as_view()),
@@ -375,7 +368,6 @@
     path('api/v2/sca_vul_summary', GetScaSummary.as_view()),
     path('api/v2/app_vul_list_content', GetAppVulsList.as_view()),
     path('api/v2/app_vul_summary', GetAppVulsSummary.as_view()),
-    path('api/v2/api_route/search', NewApiRouteSearch.as_view()),
     path('api/v2/project_version', NewProjectVersionList.as_view()),
 ])
 
diff --git a/dongtai_web/views/api_route_cover_rate.py b/dongtai_web/views/api_route_cover_rate.py
deleted file mode 100644
index 8c9e51838..000000000
--- a/dongtai_web/views/api_route_cover_rate.py
+++ /dev/null
@@ -1,87 +0,0 @@
-######################################################################
-# @author      : bidaya0 (bidaya0@$HOSTNAME)
-# @file        : api_route_cover_rate
-# @created     : Friday Aug 20, 2021 16:20:10 CST
-#
-# @description :
-######################################################################
-
-from dongtai_web.base.project_version import get_project_version, get_project_version_by_id
-from dongtai_common.models.agent import IastAgent
-from dongtai_common.endpoint import R, UserEndPoint
-from django.db.models import Q
-from django.utils.translation import gettext_lazy as _
-from dongtai_web.utils import batch_queryset, checkcover_batch
-from dongtai_web.utils import extend_schema_with_envcheck
-from dongtai_common.models.api_route import IastApiRoute, FromWhereChoices
-from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
-from rest_framework import serializers
-from dongtai_common.models.project import IastProject
-import logging
-
-logger = logging.getLogger("dongtai-webapi")
-
-
-class ApiRouteCoverRateResponseSerializer(serializers.Serializer):
-    cover_rate = serializers.IntegerField(
-        help_text=_("The api cover_rate of the project"), )
-    total_count = serializers.IntegerField(
-        help_text=_("The api cover_rate of the project"), )
-    cover_count = serializers.IntegerField(
-        help_text=_("The coverd api number  of the project"), )
-
-
-_GetResponseSerializer = get_response_serializer(
-    ApiRouteCoverRateResponseSerializer())
-
-
-class ApiRouteCoverRate(UserEndPoint):
-
-    @extend_schema_with_envcheck(
-        [{
-            'name': 'project_id',
-            'type': int
-        }, {
-            'name': 'version_id',
-            'type': int
-        }],
-        tags=[_('API Route')],
-        summary=_('API Route Coverrate'),
-        description=_("Get the API route coverrate of the project corresponding to the specified id."),
-        response_schema=_GetResponseSerializer,
-    )
-    def get(self, request):
-        project_id = request.query_params.get('project_id', None)
-        version_id = request.query_params.get('version_id', None)
-        auth_users = self.get_auth_users(request.user)
-        if not version_id:
-            current_project_version = get_project_version(project_id)
-        else:
-            current_project_version = get_project_version_by_id(version_id)
-        departments = request.user.get_relative_department()
-        projectexist = IastProject.objects.filter(department__in=departments,
-                                                  pk=project_id).first()
-        if not projectexist:
-            return R.failure(_("Parameter error"))
-        total_count = IastApiRoute.objects.filter(
-            project_id=project_id,
-            project_version_id=current_project_version.get("version_id",
-                                                           0)).count()
-        covered_count = IastApiRoute.objects.filter(
-            project_id=project_id,
-            project_version_id=current_project_version.get("version_id", 0),
-            is_cover=1).count()
-        try:
-            cover_rate = "{:.2%}".format(covered_count / total_count)
-        except ZeroDivisionError as e:
-            logger.info(e, exc_info=True)
-            cover_rate = "{:.2%}".format(1.0)
-
-        return R.success(
-            msg=_('API coverage rate obtained successfully'),
-            data={
-                'cover_rate': cover_rate,
-                "total_count": total_count,
-                "covered_count": covered_count,
-            },
-        )
diff --git a/dongtai_web/views/api_route_related_request.py b/dongtai_web/views/api_route_related_request.py
deleted file mode 100644
index b51da72f7..000000000
--- a/dongtai_web/views/api_route_related_request.py
+++ /dev/null
@@ -1,81 +0,0 @@
-######################################################################
-# @author      : bidaya0 (bidaya0@$HOSTNAME)
-# @file        : api_route_related_request
-# @created     : Saturday Aug 21, 2021 13:54:14 CST
-#
-# @description :
-######################################################################
-
-from dongtai_common.models.api_route import IastApiRoute
-from dongtai_common.models.agent_method_pool import MethodPool
-from dongtai_web.base.project_version import get_project_version, get_project_version_by_id
-from dongtai_common.endpoint import R, UserEndPoint
-from dongtai_common.models.agent import IastAgent
-from django.utils.translation import gettext_lazy as _
-from django.db.models import Q
-from dongtai_web.utils import sha1
-from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
-from rest_framework import serializers
-
-
-class ApiRouteCoverRelationSerializer(serializers.ModelSerializer):
-    class Meta:
-        model = MethodPool
-        fields = serializers.ALL_FIELDS
-
-
-_GetResponseSerializer = get_response_serializer(ApiRouteCoverRelationSerializer())
-
-
-class ApiRouteRelationRequest(UserEndPoint):
-    @extend_schema_with_envcheck(
-        [{
-            'name': 'api_route_id',
-            'type': int
-        }, {
-            'name': 'project_id',
-            'type': int
-        }, {
-            'name': 'version_id',
-            'type': int
-        }],
-        tags=[_('API Route')],
-        summary=_('API Route Relation Request'),
-        description=_("Get the coverrate of the project corresponding to the specified id."
-                      ),
-        response_schema=_GetResponseSerializer,
-    )
-    def get(self, request):
-        try:
-            page_size = int(request.query_params.get('page_size', 1))
-            page_index = int(request.query_params.get('page_index', 1))
-            api_route_id = int(request.query_params.get('api_route_id', 1))
-            api_route = IastApiRoute.objects.filter(pk=api_route_id).first()
-            if api_route is None:
-                return R.failure(msg=_("API not Fould"))
-            project_id = int(request.query_params.get('project_id', None))
-            auth_users = self.get_auth_users(request.user)
-            version_id = int(request.query_params.get('version_id', None))
-        except BaseException:
-            return R.failure(_("Parameter error"))
-        if project_id:
-            if not version_id:
-                current_project_version = get_project_version(
-                    project_id)
-            else:
-                current_project_version = get_project_version_by_id(version_id)
-            agents = IastAgent.objects.filter(
-                user__in=auth_users,
-                bind_project_id=project_id,
-                project_version_id=current_project_version.get(
-                    "version_id", 0)).values("id")
-        q = Q()
-        q = q & Q(agent_id__in=[_['id'] for _ in agents]) if project_id else q
-        q = q & Q(uri_sha1=sha1(api_route.path))
-        q = q & Q(
-            http_method__in=[_.method for _ in api_route.method.http_method.all()])
-        method = MethodPool.objects.filter(q).order_by('-update_time')[0:1].values()
-        if method:
-            return R.success(data=list(method)[0])
-        else:
-            return R.success(data={})
diff --git a/dongtai_web/views/api_route_search.py b/dongtai_web/views/api_route_search.py
deleted file mode 100644
index 322489327..000000000
--- a/dongtai_web/views/api_route_search.py
+++ /dev/null
@@ -1,348 +0,0 @@
-######################################################################
-# @author      : bidaya0 (bidaya0@$HOSTNAME)
-# @file        : api_route_search
-# @created     : Wednesday Aug 18, 2021 14:31:17 CST
-#
-# @description :
-######################################################################
-
-from django.db.models import Q
-from dongtai_common.endpoint import R, UserEndPoint
-from dongtai_common.models.api_route import (
-    IastApiRoute,
-    IastApiMethod,
-    HttpMethod,
-)
-from dongtai_common.models.agent import IastAgent
-from dongtai_web.base.project_version import get_project_version, get_project_version_by_id
-from dongtai_common.models.vulnerablity import IastVulnerabilityModel
-from django.forms.models import model_to_dict
-from functools import partial
-from dongtai_common.models.hook_type import HookType
-from rest_framework import serializers
-from django.utils.translation import gettext_lazy as _
-from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
-import logging
-from dongtai_common.models.strategy import IastStrategyModel
-from dongtai_common.models.project import IastProject
-
-logger = logging.getLogger('dongtai-webapi')
-
-
-class ApiRouteSearchRequestBodySerializer(serializers.Serializer):
-    page_size = serializers.IntegerField(
-        help_text=_("number per page"),
-        required=False,
-        default=1)
-    uri = serializers.CharField(help_text=_("The uri of the api route"),
-                                required=False)
-    http_method = serializers.CharField(
-        help_text=_("The http method of the api route"), required=False)
-    project_id = serializers.IntegerField(help_text=_("The id of the project"), )
-    version_id = serializers.IntegerField(
-        help_text=_("The version id of the project"), required=False)
-    exclude_ids = serializers.CharField(help_text=_(
-        "Exclude the api route entry with the following id, this field is used to obtain the data of the entire project in batches."
-    ),
-        required=False)
-    is_cover = serializers.ChoiceField(
-        (1, 0),
-        help_text=_("Whether the api is covered by detection, that is, there is associated request data in the record."
-                    ),
-        required=False,
-    )
-
-
-class ApiRouteHttpMethodSerialier(serializers.Serializer):
-    httpmethod = serializers.CharField()
-
-
-class ApiRouteMethodSerialier(serializers.Serializer):
-    apimethod = serializers.CharField(
-        help_text=_("The method bound to this API"))
-    httpmethods = ApiRouteHttpMethodSerialier(
-        help_text=_("The method bound to this API, in array form"), many=True)
-
-
-class ApiRouteParameterSerialier(serializers.Serializer):
-    id = serializers.IntegerField(help_text=_("The id of api route"))
-    name = serializers.CharField(help_text=_("The name of api route"))
-    parameter_type = serializers.CharField(
-        help_text=_("The type of the parameter"))
-    parameter_type_shortcut = serializers.CharField(help_text=_(
-        "The shortcut of the parameter_type,e.g. java.lang.String -> String"))
-    annotaion = serializers.CharField(
-        help_text=_("The annotaion of the parameter"))
-    route = serializers.IntegerField(help_text=_("The route id of parameter"))
-
-
-class ApiRouteResponseSerialier(serializers.Serializer):
-    id = serializers.IntegerField(help_text=_("The id of api response"))
-    return_type = serializers.CharField(
-        help_text=_("The return type of api route"))
-    route = serializers.IntegerField(
-        help_text=_("The route id of api response"))
-    return_type_shortcut = serializers.CharField(
-        help_text=_("The shortcut of return_type"))
-
-
-class ApiRouteVulnerabitySerialier(serializers.Serializer):
-    level_id = serializers.IntegerField(
-        help_text=_("The vulnerablity level id "))
-    hook_type_name = serializers.CharField(
-        help_text=_("The vulnerablity type name"))
-
-
-class ApiRouteSearchResponseSerializer(serializers.Serializer):
-    id = serializers.IntegerField(help_text=_("The id of api route"))
-    path = serializers.CharField(help_text=_("The uri of api route"))
-    code_class = serializers.CharField(help_text=_("The class of api route"))
-    description = serializers.CharField(
-        help_text=_("The description of the api route"))
-    code_file = serializers.CharField(
-        help_text=_("The code file of the api route"))
-    controller = serializers.CharField(
-        help_text=_("The controller of the api route"))
-    agent = serializers.IntegerField(
-        help_text=_("The id of the agent reported the api route"))
-    is_cover = serializers.ChoiceField(
-        (1, 0),
-        help_text=_("Whether the api is covered by detection, that is, there is associated request data in the record."
-                    ),
-        required=False,
-    )
-    responses = ApiRouteResponseSerialier(many=True)
-    parameters = ApiRouteParameterSerialier(many=True)
-    vulnerablities = ApiRouteVulnerabitySerialier(many=True)
-    method = ApiRouteMethodSerialier()
-
-
-_GetResponseSerializer = get_response_serializer(
-    ApiRouteSearchResponseSerializer())
-
-
-class ApiRouteSearch(UserEndPoint):
-    @extend_schema_with_envcheck(
-        request=ApiRouteSearchRequestBodySerializer,
-        tags=[_('API Route')],
-        summary=_('API Route Search'),
-        description=_("Get the API list corresponding to the project according to the following parameters. By default, there is no sorting. Please use the exclude_ids field for pagination."
-                      ),
-        response_schema=_GetResponseSerializer,
-    )
-    def post(self, request):
-        try:
-            page_size = int(request.data.get('page_size', 1))
-            page_index = int(request.data.get('page_index', 0))
-            uri = request.data.get('uri', None)
-            http_method = request.data.get('http_method', None)
-            project_id = request.data.get('project_id', None)
-            project_id = int(project_id) if project_id else None
-            version_id = request.data.get('version_id', None)
-            version_id = int(version_id) if version_id else None
-            exclude_id = request.data.get('exclude_ids', None)
-            exclude_id = [int(i)
-                          for i in exclude_id.split(',')] if exclude_id else None
-            is_cover = request.data.get('is_cover', None)
-            is_cover_dict = {1: True, 0: False}
-            is_cover = is_cover_dict[int(is_cover)] if is_cover is not None and is_cover != '' else None
-        except Exception as e:
-            logger.error(e)
-            return R.failure(_("Parameter error"))
-        auth_users = self.get_auth_users(request.user)
-
-        if http_method:
-            http_method_obj = HttpMethod.objects.filter(method=http_method.upper())[0:1]
-            if http_method_obj:
-                api_methods = IastApiMethod.objects.filter(
-                    http_method__id=http_method_obj[0].id).all().values('id')
-            else:
-                api_methods = []
-        else:
-            api_methods = []
-
-        if not version_id:
-            current_project_version = get_project_version(
-                project_id)
-        else:
-            current_project_version = get_project_version_by_id(version_id)
-        departments = request.user.get_relative_department()
-        agents = IastAgent.objects.filter(
-            bind_project__department__in=departments,
-            bind_project_id=project_id,
-            project_version_id=current_project_version.get("version_id",
-                                                           0)).values("id")
-        q = Q(agent_id__in=[_['id'] for _ in agents])
-        projectexist = IastProject.objects.filter(department__in=departments,
-                                                  pk=project_id).first()
-        if not projectexist:
-            return R.failure(_("Parameter error"))
-        agents = IastAgent.objects.filter(
-            bind_project_id=project_id,
-            project_version_id=current_project_version.get("version_id",
-                                                           0)).values("id")
-        q = Q(project_version_id=current_project_version.get("version_id", 0),
-              project_id=project_id)
-        q = q & Q(
-            method_id__in=[_['id']
-                           for _ in api_methods]) if api_methods != [] else q
-        q = q & Q(path__icontains=uri) if uri else q
-        q = q & ~Q(pk__in=exclude_id) if exclude_id and not page_index else q
-        q = q & Q(is_cover=is_cover) if is_cover is not None else q
-        api_routes = IastApiRoute.objects.filter(q).order_by(
-            'id').select_related('method').prefetch_related(
-                'iastapiresponse_set', 'iastapiparameter_set').all()
-        if page_index:
-            no_used, api_routes = self.get_paginator(api_routes, page_index,
-                                                     page_size)
-        distinct_fields = ["path", "method_id"]
-        distinct_exist_list = [] if not exclude_id else list(
-            set(
-                filter(lambda x: x != '', [
-                    distinct_key(
-                        IastApiRoute.objects.filter(pk=i).values(
-                            "path", "method_id").first(), distinct_fields)
-                    for i in exclude_id
-                ])))
-        _filter_and_label_partial = partial(
-            _filter_and_label,
-            distinct=True,
-            distinct_fields=distinct_fields,
-            distinct_exist_list=distinct_exist_list)
-        api_routes = _filter_and_label_partial(
-            api_routes, page_size, agents, http_method,
-            is_cover) if is_cover is not None else _filter_and_label_partial(
-                api_routes, page_size, agents, http_method)
-        return R.success(
-            data=[_serialize(api_route, agents) for api_route in api_routes])
-
-
-def _filter_and_label(api_routes,
-                      limit,
-                      agents,
-                      http_method,
-                      is_cover=None,
-                      distinct=True,
-                      distinct_fields=['path', 'method_id'],
-                      distinct_exist_list=[]):
-    api_routes_after_filter = []
-    distinct_exist_list = distinct_exist_list.copy()
-    for api_route in api_routes:
-        distinct_key_ = distinct_key(
-            {
-                'path': api_route.path,
-                'method_id': api_route.method.id
-            }, distinct_fields)
-        if distinct_key_ in distinct_exist_list:
-            continue
-        else:
-            distinct_exist_list.append(distinct_key_)
-#        api_route.is_cover = checkcover(api_route, agents, http_method)
-        if is_cover is not None:
-            api_routes_after_filter += [
-                api_route
-            ] if api_route.is_cover == is_cover else []
-        else:
-            api_routes_after_filter += [api_route]
-        if limit == len(api_routes_after_filter):
-            break
-    return api_routes_after_filter
-
-
-def distinct_key(objects, fields):
-    if objects is None:
-        return ''
-    sequence = [objects.get(field, 'None') for field in fields]
-    sequence = [
-        item if isinstance(item, str) else str(item) for item in sequence
-    ]
-    return '_'.join(sequence)
-
-
-def _serialize(api_route, agents):
-    item = model_to_dict(api_route)
-    is_cover_dict = {1: True, 0: False}
-    is_cover_dict = _inverse_dict(is_cover_dict)
-    item['is_cover'] = is_cover_dict[api_route.is_cover]
-    item['parameters'] = _get_parameters(api_route)
-    item['responses'] = _get_responses(api_route)
-    item['method'] = _get_api_method(api_route.method)
-    item['vulnerablities'] = _get_vuls(item['path'], agents)
-    return item
-
-
-def serialize(api_route):
-    item = model_to_dict(api_route)
-    item['parameters'] = _get_parameters(api_route)
-    item['responses'] = _get_responses(api_route)
-    item['method'] = _get_api_method(item['method'])
-    return item
-
-
-def _get_vuls(uri, agents):
-    vuls = IastVulnerabilityModel.objects.filter(
-        uri=uri, agent_id__in=[_['id'] for _ in agents],
-        is_del=0).values('hook_type_id', 'level_id', 'strategy_id',
-                         'strategy__vul_name').distinct().all()
-    return [_get_hook_type(dict(vul)) for vul in vuls]
-
-
-def _get_hook_type(vul: dict) -> dict:
-    return {
-        'hook_type_name': vul['strategy__vul_name'],
-        'level_id': vul['level_id']
-    }
-
-    hook_type = HookType.objects.filter(pk=vul['hook_type_id']).first()
-    hook_type_name = hook_type.name if hook_type else None
-    strategy = IastStrategyModel.objects.filter(pk=vul['strategy_id']).first()
-    strategy_name = strategy.vul_name if strategy else None
-    type_ = list(
-        filter(lambda x: x is not None, [strategy_name, hook_type_name]))
-    type_name = type_[0] if type_ else ''
-    return {'hook_type_name': type_name, 'level_id': vul['level_id']}
-
-
-def _get_parameters(api_route):
-    parameters = api_route.iastapiparameter_set.all()
-    #    parameters = IastApiParameter.objects.filter(route=api_route).all()
-    parameters = [model_to_dict(parameter) for parameter in parameters]
-    parameters = [_get_parameters_type(parameter) for parameter in parameters]
-    return parameters
-
-
-def _get_parameters_type(api_route):
-    api_route['parameter_type_shortcut'] = api_route['parameter_type'].split(
-        '.')[-1]
-    return api_route
-
-
-def _get_responses(api_route):
-    responses = api_route.iastapiresponse_set.all()
-    #    responses = IastApiResponse.objects.filter(route=api_route).all()
-    responses = [model_to_dict(response) for response in responses]
-    responses = [_get_responses_type(response) for response in responses]
-    return responses
-
-
-def _get_responses_type(api_route):
-    api_route['return_type_shortcut'] = api_route['return_type'].split('.')[-1]
-    return api_route
-
-
-def _get_api_method(api_method):
-    apimethod = api_method.http_method.all()
-    res = {}
-    res['apimethod'] = api_method.method
-    res['httpmethods'] = [_.method for _ in api_method.http_method.all()]
-    return res
-#    if apimethod:
-#        res = {}
-#        res['apimethod'] = apimethod.method
-#        res['httpmethods'] = [_.method for _ in apimethod.http_method.all()]
-#        return res
-#    return {}
-
-
-def _inverse_dict(dic: dict) -> dict:
-    return {v: k for k, v in dic.items()}
diff --git a/dongtai_web/views/new_project_query.py b/dongtai_web/views/new_project_query.py
index 69ba90f6c..a1dfbe38b 100644
--- a/dongtai_web/views/new_project_query.py
+++ b/dongtai_web/views/new_project_query.py
@@ -4,18 +4,17 @@
 # software: PyCharm
 # project: lingzhi-webapi
 import logging
-import time
-from dongtai_common.endpoint import R
+
 from django.db.models import Q
 from django.forms.models import model_to_dict
-from dongtai_common.endpoint import UserEndPoint
-from dongtai_common.models.project_version import IastProjectVersion
-from dongtai_common.models.api_route import IastApiRoute
 from django.utils.translation import gettext_lazy as _
-from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
 from rest_framework import serializers
 from rest_framework.serializers import ValidationError
 
+from dongtai_common.endpoint import R, UserEndPoint
+from dongtai_common.models.project_version import IastProjectVersion
+from dongtai_web.utils import extend_schema_with_envcheck
+
 logger = logging.getLogger("django")
 
 
@@ -31,24 +30,6 @@ class ProjectVersionArgSerializer(serializers.Serializer):
                                          required=False)
 
 
-class ApiRouteArgSerializer(serializers.Serializer):
-    page_size = serializers.IntegerField(default=20,
-                                         help_text=_('Number per page'))
-    page = serializers.IntegerField(default=1, help_text=_('Page index'))
-    version_id = serializers.IntegerField(default=None,
-                                          help_text=_('Project id'),
-                                          required=False)
-    project_id = serializers.IntegerField(default=None,
-                                          help_text=_('Project id'),
-                                          required=False)
-    is_cover = serializers.IntegerField(default=None,
-                                        help_text=_('Project id'),
-                                        required=False)
-    from_where = serializers.IntegerField(default=None,
-                                          help_text=_('Project id'),
-                                          required=False)
-
-
 class NewProjectVersionList(UserEndPoint):
     name = "api-v1-project-version-delete"
     description = _("Delete application version information")
@@ -80,48 +61,3 @@ def get(self, request):
         return R.success(
             data=[model_to_dict(document) for document in documents],
             page=page_info)
-
-
-class NewApiRouteSearch(UserEndPoint):
-    name = "api-v1-api-route-search"
-    description = _("Delete application version information")
-
-    @extend_schema_with_envcheck(
-        request=ApiRouteArgSerializer,
-        tags=[_('API Route')],
-        summary=_('New api route search'),
-        description=_("Get the item corresponding to the user, support fuzzy search based on name."),
-    )
-    def post(self, request):
-        ser = ApiRouteArgSerializer(data=request.data)
-        try:
-            if ser.is_valid(True):
-                page_size = ser.validated_data['page_size']
-                page = ser.validated_data['page']
-                project_id = ser.validated_data['project_id']
-                version_id = ser.validated_data['version_id']
-                is_cover = ser.validated_data['is_cover']
-                from_where = ser.validated_data['from_where']
-        except ValidationError as e:
-            return R.failure(data=e.detail)
-        q = Q()
-        if project_id:
-            q = q & Q(project_id=project_id)
-        if version_id:
-            q = q & Q(project_version_id=version_id)
-        if is_cover is not None:
-            q = q & Q(is_cover=is_cover)
-        if from_where is not None:
-            q = q & Q(from_where=from_where)
-
-        page_info, documents = self.get_paginator(
-            IastApiRoute.objects.filter(q).order_by('-id').values(
-                'method__method', 'path', 'id', 'project_id',
-                'project_version', 'controller', 'code_class', 'code_file',
-                'is_cover').all(), page, page_size)
-        documents = list(documents)
-        for document in documents:
-            document['method'] = {
-                "httpmethods": document['method__method'].split("/")
-            }
-        return R.success(data=documents, page=page_info)

From 8c05cb5cc831a44c5a953a7d0803556cb797a2df Mon Sep 17 00:00:00 2001
From: st1020 
Date: Fri, 7 Jul 2023 15:03:04 +0800
Subject: [PATCH 071/161] feat: remove agent log download api

---
 dongtai_web/urls.py               |   9 --
 dongtai_web/views/log_download.py | 188 ------------------------------
 2 files changed, 197 deletions(-)
 delete mode 100644 dongtai_web/views/log_download.py

diff --git a/dongtai_web/urls.py b/dongtai_web/urls.py
index b017a94e7..208bc2d26 100644
--- a/dongtai_web/urls.py
+++ b/dongtai_web/urls.py
@@ -127,7 +127,6 @@
 from dongtai_web.threshold.get_webhook_setting import GetAgentWebHookConfig
 from dongtai_web.threshold.webhook_type import AgentWebHookTypeList
 from dongtai_web.threshold.get_config_setting import GetAgentThresholdConfig
-from dongtai_web.views.log_download import AgentLogDownload
 
 from dongtai_web.threshold.agent_core_status import (AgentCoreStatusUpdate,
                                                      AgentCoreStatusUpdateALL)
@@ -284,14 +283,6 @@
     path('agent/core/update/all', AgentCoreStatusUpdateALL.as_view()),
     path('agent/summary/', AgentSummary.as_view()),
 
-    # 消息通知规则配置
-    path('agent/log/batch',
-         AgentLogDownload.as_view({'post': 'batch_task_add'})),
-    path('agent/log/', AgentLogDownload.as_view({'get':
-                                                         'get_single'})),
-    path('agent/log/batch/',
-         AgentLogDownload.as_view({'get': 'batch_log_download'})),
-
     # vul list page of sca and common vul
     path('vul_list_delete', DelVulMany.as_view()),
     path('project_vul_delete', DelVulProjectLevel.as_view()),
diff --git a/dongtai_web/views/log_download.py b/dongtai_web/views/log_download.py
deleted file mode 100644
index 8a2f4d9b2..000000000
--- a/dongtai_web/views/log_download.py
+++ /dev/null
@@ -1,188 +0,0 @@
-import os
-import logging
-import zipfile
-from dongtai_common.endpoint import UserEndPoint
-from io import BytesIO
-from dongtai_common.models.agent import IastAgent
-from enum import Enum
-from django.http import FileResponse
-from rest_framework import viewsets
-from result import Ok, Err, Result
-from functools import partial
-from wsgiref.util import FileWrapper
-from django.http import HttpResponseNotFound
-from dongtai_common.models.message import IastMessage
-import threading
-from django.db.models import Q
-from django.db import transaction
-from dongtai_conf.settings import TMP_COMMON_PATH
-from tempfile import NamedTemporaryFile
-from dongtai_common.endpoint import R
-from drf_spectacular.utils import extend_schema
-from django.utils.translation import gettext_lazy as _
-
-logger = logging.getLogger('dongtai-webapi')
-
-
-class ResultType(Enum):
-    OK = 1
-    ERR = 2
-
-
-def nothing_resp():
-    return HttpResponseNotFound("找不到相关日志数据")
-
-
-class AgentLogDownload(UserEndPoint, viewsets.ViewSet):
-
-    def get_single(self, request, pk):
-        try:
-            a = int(pk) > 0
-            if not a:
-                return nothing_resp()
-        except BaseException:
-            return nothing_resp()
-        department = request.user.get_relative_department()
-        if IastAgent.objects.filter(pk=pk,
-                                    department__in=department).exists():
-            result = get_newest_log_zip(pk)
-            if isinstance(result, Err):
-                return nothing_resp()
-            file_ = result.value
-            file_.seek(0)
-            response = FileResponse(FileWrapper(file_))
-            response['content_type'] = 'application/octet-stream'
-            response['Content-Disposition'] = f"attachment; filename={pk}.zip"
-            return response
-        return nothing_resp()
-
-    @extend_schema(
-        tags=[_('Agent')],
-        summary="探针日志批量下载",
-    )
-    def batch_task_add(self, request):
-        mode = request.data.get('mode', 1)
-        department = request.user.get_relative_department()
-        q = Q(department__in=department)
-        if mode == 1:
-            ids = request.data.get('ids', [])
-            q = q & Q(pk__in=ids)
-        elif mode == 2:
-            q = q
-
-        def generate_zip_thread():
-            generate_agent_log_zip(q, request.user.id)
-
-        t1 = threading.Thread(target=generate_zip_thread, daemon=True)
-        t1.start()
-        return R.success()
-
-    @extend_schema(
-        tags=[_('Agent')],
-        summary="探针日志下载",
-    )
-    def batch_log_download(self, request, pk):
-        try:
-            a = int(pk) > 0
-            if not a:
-                return nothing_resp()
-            return FileResponse(open(
-                os.path.join(TMP_COMMON_PATH, f'batchagent/{pk}.zip'), 'rb'),
-                filename='agentlog.zip')
-        except FileNotFoundError as e:
-            logger.info(e)
-            return nothing_resp()
-        except Exception as e:
-            logger.info(e)
-            return nothing_resp()
-
-
-def generate_path(agent_id):
-    return os.path.join(TMP_COMMON_PATH, f'agent/{agent_id}/')
-
-
-def get_newest_log_zip(agent_id: int) -> Result:
-    path = generate_path(agent_id)
-    res = file_newest_2_file_under_path(path)
-    if isinstance(res, Err):
-        return res
-    res = getzipfilesinmemorty(res.value)
-    return res
-
-
-def getzipfilesinmemorty(filenames: list) -> Result[BytesIO, str]:
-    try:
-        zip_subdir = "logs"
-        s = BytesIO()
-        with zipfile.ZipFile(s, "w") as zf:
-            for fpath in filenames:
-                fdir, fname = os.path.split(fpath)
-                zip_path = os.path.join(zip_subdir, fname)
-                zf.write(fpath, zip_path)
-            zf.close()
-        return Ok(s)
-    except Exception as e:
-        logger.error(e, exc_info=True)
-        return Err('unexcept eror')
-
-
-def file_newest_N_file_under_path(path: str, N: int) -> Result[list[str], str]:
-    try:
-        files = [
-            f for f in os.listdir(path)
-            if os.path.isfile(os.path.join(path, f))
-        ]
-        paths = [os.path.join(path, basename) for basename in files]
-        return Ok(sorted(paths, key=os.path.getctime, reverse=True)[:N])
-    except (FileNotFoundError, ValueError) as e:
-        return Err('file path error')
-    except Exception as e:
-        logger.error(e, exc_info=True)
-        return Err('unexcept error')
-
-
-file_newest_file_under_path = partial(file_newest_N_file_under_path, N=1)
-file_newest_2_file_under_path = partial(file_newest_N_file_under_path, N=2)
-
-
-def zip_file_write(msg_id, items):
-    from zipfile import ZipFile
-    zipfilepath = os.path.join(TMP_COMMON_PATH, f'batchagent/{msg_id}.zip')
-    zip_subdir = "logs"
-    with ZipFile(zipfilepath, 'w') as zipObj:
-        with NamedTemporaryFile() as tmpfile:
-            zipObj.write(tmpfile.name)
-        for i in items:
-            for k in i:
-                path1, filename = os.path.split(k)
-                path2, agent_id = os.path.split(path1)
-                zipObj.write(
-                    k, os.path.join(zip_subdir, f'/{agent_id}/', filename))
-    return zipfilepath
-
-
-def get_zip_together(agents_ids, msg_id):
-    from zipfile import ZipFile
-    res = map(
-        lambda x: x.value,
-        filter(
-            lambda x: isinstance(x, Ok),
-            map(file_newest_2_file_under_path, map(generate_path,
-                                                   agents_ids))))
-    return zip_file_write(msg_id, res)
-
-
-@transaction.atomic
-def generate_agent_log_zip(q, user_id):
-    agent_ids = IastAgent.objects.filter(q).values_list('id', flat=True)
-    msg = IastMessage.objects.create(message='AGENT日志导出成功',
-                                     message_type_id=2,
-                                     relative_url='/api/v1/agent/log/tmp',
-                                     to_user_id=user_id)
-
-    zip_file_size = os.path.getsize(get_zip_together(agent_ids, msg.id))
-    msg.relative_url = f'/api/v1/agent/log/batch/{msg.id}'
-    if int(zip_file_size) < 500:
-        msg.message = 'agent日志获取失败，请登录项目服务器获取'
-        msg.relative_url = f'/api/v1/agent/log/batch/null'
-    msg.save()

From 9f4afa4c07bc52f2c58a9df7f68c2d92e8614fc6 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Fri, 7 Jul 2023 15:14:55 +0800
Subject: [PATCH 072/161] feat: remove agent update api

---
 dongtai_web/threshold/agent_core_status.py | 123 ---------------------
 dongtai_web/urls.py                        |   4 -
 2 files changed, 127 deletions(-)
 delete mode 100644 dongtai_web/threshold/agent_core_status.py

diff --git a/dongtai_web/threshold/agent_core_status.py b/dongtai_web/threshold/agent_core_status.py
deleted file mode 100644
index fe44f2b9c..000000000
--- a/dongtai_web/threshold/agent_core_status.py
+++ /dev/null
@@ -1,123 +0,0 @@
-#!/usr/bin/env python
-# -*- coding:utf-8 -*-
-# author:sjh
-# software: PyCharm
-import time
-from dongtai_common.endpoint import UserEndPoint, R
-
-from dongtai_common.models.agent import IastAgent
-from django.utils.translation import gettext_lazy as _
-from rest_framework import serializers
-from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
-from dongtai_web.serializers.agent import AgentToggleArgsSerializer
-from dongtai_web.views import AGENT_STATUS
-from collections import defaultdict
-from dongtai_common.utils.const import OPERATE_PUT
-
-STATUS_MAPPING = defaultdict(lambda: 1, {3: 1, 4: 2})
-
-
-_ResponseSerializer = get_response_serializer(
-    status_msg_keypair=(((201, _('Suspending ...')), ''), ))
-
-
-class AgentCoreStatusSerializer(serializers.Serializer):
-
-    id = serializers.IntegerField(help_text=_('The id of the webHook.'), required=False)
-    core_status = serializers.IntegerField(help_text=_('The type of the webHook.'), required=True)
-    agent_ids = serializers.CharField(help_text=_('The cluster_name of the agent.'), max_length=255, required=False)
-
-
-class AgentCoreStatusUpdate(UserEndPoint):
-    name = "api-v1-agent-core-status-update"
-    description = _("Suspend Agent")
-
-    @extend_schema_with_envcheck(
-        request=AgentToggleArgsSerializer,
-        tags=[
-            _('Agent'),
-            OPERATE_PUT,
-        ],
-        deprecated=True,
-        summary=_('Agent Status Update'),
-        description=_("Control the running agent by specifying the id."),
-        response_schema=_ResponseSerializer,
-    )
-    def post(self, request):
-        ser = AgentCoreStatusSerializer(data=request.data)
-        if ser.is_valid(False):
-            agent_id = ser.validated_data.get('id', None)
-            core_status = ser.validated_data.get('core_status', None)
-            agent_ids = ser.validated_data.get('agent_ids', "").strip()
-        else:
-            return R.failure(msg=_('Incomplete parameter, please check again'))
-
-        if agent_ids:
-            try:
-                agent_ids = [int(i) for i in agent_ids.split(',')]
-            except BaseException:
-                return R.failure(_("Parameter error"))
-        elif agent_id is not None:
-            agent_ids = [int(agent_id)]
-
-        if agent_ids:
-            department = request.user.get_relative_department()
-            except_running_status = STATUS_MAPPING[core_status]
-            # Here could be simply to such as "control_status in statusData.keys()"
-            statusData = AGENT_STATUS.get(core_status, {})
-            control_status = statusData.get("value", None)
-            if control_status is None:
-                return R.failure(msg=_('Incomplete parameter, please check again'))
-
-            queryset = IastAgent.objects.filter(department__in=department)
-            queryset.filter(id__in=agent_ids).update(
-                except_running_status=except_running_status,
-                control=core_status,
-                is_control=1,
-                latest_time=int(time.time()))
-            # for agent_id in agent_ids:
-            #     agent = IastAgent.objects.filter(user=request.user, id=agent_id).first()
-            #     if agent is None:
-            #         continue
-            #     # edit by song
-            #     # if agent.is_control == 1 and agent.control != 3 and agent.control != 4:
-            #     #     continue
-            #     agent.control = core_status
-            #     agent.is_control = 1
-            #     agent.latest_time = int(time.time())
-            #     agent.save(update_fields=['latest_time', 'control', 'is_control'])
-
-        return R.success(msg=_('状态已下发'))
-
-
-class AgentCoreStatusUpdateALL(UserEndPoint):
-    name = "api-v1-agent-core-status-update"
-    description = _("Suspend Agent")
-
-    @extend_schema_with_envcheck(
-        request=AgentToggleArgsSerializer,
-        tags=[
-            _('Agent'),
-            OPERATE_PUT,
-        ],
-        deprecated=True,
-        summary="Agent 批量更新",
-        description=_("Control the running agent by specifying the id."),
-        response_schema=_ResponseSerializer,
-    )
-    def post(self, request):
-        ser = AgentCoreStatusSerializer(data=request.data)
-        department = request.user.get_relative_department()
-        if ser.is_valid(False):
-            core_status = ser.validated_data.get('core_status', None)
-        else:
-            return R.failure(msg=_('Incomplete parameter, please check again'))
-
-        #queryset = IastAgent.objects.filter(department__in=department)
-        #except_running_status = STATUS_MAPPING[core_status]
-        #queryset.filter(online=1).update(
-        #    except_running_status=except_running_status,
-        #    control=core_status,
-        #    is_control=1,
-        #    latest_time=int(time.time()))
-        return R.success(msg=_('状态已下发'))
diff --git a/dongtai_web/urls.py b/dongtai_web/urls.py
index 208bc2d26..b34e92aa5 100644
--- a/dongtai_web/urls.py
+++ b/dongtai_web/urls.py
@@ -128,8 +128,6 @@
 from dongtai_web.threshold.webhook_type import AgentWebHookTypeList
 from dongtai_web.threshold.get_config_setting import GetAgentThresholdConfig
 
-from dongtai_web.threshold.agent_core_status import (AgentCoreStatusUpdate,
-                                                     AgentCoreStatusUpdateALL)
 from dongtai_web.aggregation.aggregation_del import DelVulMany
 from dongtai_web.aggregation.aggregation_project_del import DelVulProjectLevel
 
@@ -279,8 +277,6 @@
 
     # get webHook setting
     path('webhook/settings/get', GetAgentWebHookConfig.as_view()),
-    path('agent/core/update', AgentCoreStatusUpdate.as_view()),
-    path('agent/core/update/all', AgentCoreStatusUpdateALL.as_view()),
     path('agent/summary/', AgentSummary.as_view()),
 
     # vul list page of sca and common vul

From d8fbb655ed8cb083e243256fc8e88cc923ac4087 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Fri, 7 Jul 2023 15:40:00 +0800
Subject: [PATCH 073/161] fix: fix test error

---
 dongtai_web/tests.py | 20 +-------------------
 1 file changed, 1 insertion(+), 19 deletions(-)

diff --git a/dongtai_web/tests.py b/dongtai_web/tests.py
index 205e77a37..489c51ce3 100644
--- a/dongtai_web/tests.py
+++ b/dongtai_web/tests.py
@@ -1,9 +1,7 @@
 # Create your tests here.
 from django.test import TestCase
 from dongtai_web.views.agents_v2 import query_agent
-from dongtai_web.views.log_download import (file_newest_N_file_under_path, getzipfilesinmemorty, )
-from dongtai_web.views.agents_v2 import (
-    query_agent, )
+from dongtai_web.views.agents_v2 import query_agent
 
 from dongtai_web.threshold.config_setting import (convert_choices_to_dict,
                                                   convert_choices_to_value_dict,
@@ -27,22 +25,6 @@ def test_query_agent(self):
         print(res)
 
 
-class ZipFileTestCase(TestCase):
-
-    def test_findnewest_file(self):
-        res = file_newest_N_file_under_path('./dongtai_web', 2)
-        print(res)
-
-    def test_getzipfilesinmemorty(self):
-        res = getzipfilesinmemorty(['./README.md', 'Pipfile'])
-        print(res)
-
-    def test_get_zip_together(self):
-        from dongtai_web.views.log_download import get_zip_together
-        res = get_zip_together([1, 2, 3], 1)
-        print(res)
-
-
 class ChoiceConvertTestCase(TestCase):
     def test_choice_convert(self):
         able_to_search = (MetricType, MetricGroup,

From f4da1457e2894258d60a4fdaead7c22c887a4bcf Mon Sep 17 00:00:00 2001
From: st1020 
Date: Fri, 7 Jul 2023 15:42:47 +0800
Subject: [PATCH 074/161] fix: fix test error

---
 dongtai_web/tests.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/dongtai_web/tests.py b/dongtai_web/tests.py
index 489c51ce3..8adc6ba5d 100644
--- a/dongtai_web/tests.py
+++ b/dongtai_web/tests.py
@@ -1,7 +1,6 @@
 # Create your tests here.
 from django.test import TestCase
 from dongtai_web.views.agents_v2 import query_agent
-from dongtai_web.views.agents_v2 import query_agent
 
 from dongtai_web.threshold.config_setting import (convert_choices_to_dict,
                                                   convert_choices_to_value_dict,

From edde813efa3164a0d7ea7569092907d19aa79bf1 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Fri, 7 Jul 2023 16:44:09 +0800
Subject: [PATCH 075/161] feat: remove threshold api

---
 dongtai_web/tests.py                          |  55 ---
 dongtai_web/threshold/__init__.py             |   0
 dongtai_web/threshold/config_setting.py       | 425 ------------------
 .../threshold/del_threshold_setting.py        |  42 --
 dongtai_web/threshold/del_webhook_setting.py  |  42 --
 dongtai_web/threshold/get_config_setting.py   |  59 ---
 .../threshold/get_config_setting_detail.py    |  37 --
 dongtai_web/threshold/get_webhook_setting.py  |  42 --
 dongtai_web/threshold/tests.py                |  18 -
 dongtai_web/threshold/webhook_setting.py      |  68 ---
 dongtai_web/threshold/webhook_type.py         |  75 ----
 dongtai_web/urls.py                           |  42 --
 12 files changed, 905 deletions(-)
 delete mode 100644 dongtai_web/tests.py
 delete mode 100644 dongtai_web/threshold/__init__.py
 delete mode 100644 dongtai_web/threshold/config_setting.py
 delete mode 100644 dongtai_web/threshold/del_threshold_setting.py
 delete mode 100644 dongtai_web/threshold/del_webhook_setting.py
 delete mode 100644 dongtai_web/threshold/get_config_setting.py
 delete mode 100644 dongtai_web/threshold/get_config_setting_detail.py
 delete mode 100644 dongtai_web/threshold/get_webhook_setting.py
 delete mode 100644 dongtai_web/threshold/tests.py
 delete mode 100644 dongtai_web/threshold/webhook_setting.py
 delete mode 100644 dongtai_web/threshold/webhook_type.py

diff --git a/dongtai_web/tests.py b/dongtai_web/tests.py
deleted file mode 100644
index 8adc6ba5d..000000000
--- a/dongtai_web/tests.py
+++ /dev/null
@@ -1,55 +0,0 @@
-# Create your tests here.
-from django.test import TestCase
-from dongtai_web.views.agents_v2 import query_agent
-
-from dongtai_web.threshold.config_setting import (convert_choices_to_dict,
-                                                  convert_choices_to_value_dict,
-                                                  get_metric_types, get_targets)
-from dongtai_common.models.agent_config import (
-    IastCircuitTarget,
-    IastCircuitConfig,
-    IastCircuitMetric,
-    TargetType,
-    TargetOperator,
-    DealType,
-    MetricType,
-    MetricGroup,
-    MetricOperator,
-)
-
-
-class DashboardTestCase(TestCase):
-    def test_query_agent(self):
-        res = query_agent()
-        print(res)
-
-
-class ChoiceConvertTestCase(TestCase):
-    def test_choice_convert(self):
-        able_to_search = (MetricType, MetricGroup,
-                          TargetOperator,
-                          MetricOperator)
-        for i in able_to_search:
-            res = convert_choices_to_dict(i)
-            print(res)
-
-    def test_choice_convert_value(self):
-        able_to_search = (MetricType, MetricGroup,
-                          TargetOperator,
-                          MetricOperator)
-        for i in able_to_search:
-            res = convert_choices_to_value_dict(i)
-            print(res)
-
-    def test_metric_string_concate(self):
-        metrics = [{
-            "metric_type": 1,
-            "opt": 5,
-            "value": 100
-        }, {
-            "metric_type": 2,
-            "opt": 5,
-            "value": 100
-        }]
-        res = get_metric_types(metrics)
-        print(res)
diff --git a/dongtai_web/threshold/__init__.py b/dongtai_web/threshold/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/dongtai_web/threshold/config_setting.py b/dongtai_web/threshold/config_setting.py
deleted file mode 100644
index 7a5a7a437..000000000
--- a/dongtai_web/threshold/config_setting.py
+++ /dev/null
@@ -1,425 +0,0 @@
-#!/usr/bin/env python
-# -*- coding:utf-8 -*-
-# author:sjh
-# software: PyCharm
-# project: webApi
-# agent threshold setting
-from dongtai_conf.settings import DEFAULT_CIRCUITCONFIG
-from django.db.models import F
-from django.forms.models import model_to_dict
-from django.db.models import Max, Min
-from inflection import underscore
-from collections.abc import Iterable
-from dongtai_common.models.agent_config import (
-    IastCircuitTarget,
-    IastCircuitConfig,
-    IastCircuitMetric,
-    TargetType,
-    TargetOperator,
-    DealType,
-    MetricType,
-    MetricGroup,
-    MetricOperator,
-    SystemMetricType,
-    JVMMetricType,
-    ApplicationMetricType,
-    UNIT_DICT,
-)
-from rest_framework import viewsets
-from rest_framework import serializers
-import time
-
-from dongtai_common.endpoint import UserEndPoint, R, TalentAdminEndPoint
-from dongtai_common.models.agent_config import IastAgentConfig
-from django.utils.translation import gettext_lazy as _
-from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
-from dongtai_web.serializers.agent_config import AgentConfigSettingSerializer
-from rest_framework.serializers import ValidationError
-from rest_framework.utils.serializer_helpers import ReturnDict
-from typing import Dict
-from collections import OrderedDict
-
-_ResponseSerializer = get_response_serializer(status_msg_keypair=(
-    ((201, _('The setting is complete')), ''),
-    ((202, _('Incomplete parameter, please try again later')), '')
-))
-
-
-class AgentThresholdConfig(UserEndPoint):
-    name = "api-v1-agent-threshold-config-setting"
-    description = _("config Agent")
-
-    def create_agent_config(self, user, details, hostname, ip, port, cluster_name, cluster_version, priority, id):
-        try:
-
-            timestamp = int(time.time())
-            if id:
-                strategy = IastAgentConfig.objects.filter(user=user, id=id).order_by("-create_time").first()
-            else:
-                strategy = IastAgentConfig.objects.filter(user=user, id=id).order_by("-create_time").first()
-            if strategy:
-                strategy.details = details
-                strategy.hostname = hostname
-                strategy.ip = ip
-                strategy.port = port
-                strategy.cluster_name = cluster_name
-                strategy.cluster_version = cluster_version
-                strategy.priority = priority
-            else:
-                strategy = IastAgentConfig(
-                    user=user,
-                    details=details,
-                    hostname=hostname,
-                    ip=ip,
-                    port=port,
-                    cluster_name=cluster_name,
-                    cluster_version=cluster_version,
-                    priority=priority,
-                    create_time=timestamp
-                )
-            strategy.save()
-            return strategy
-        except Exception as e:
-
-            return None
-
-    @extend_schema_with_envcheck(
-        tags=[_('Agent')],
-        summary=_('Agent threshold Config'),
-        description=_("Configure agent disaster recovery strategy"),
-        response_schema=_ResponseSerializer)
-    def post(self, request):
-
-        ser = AgentConfigSettingSerializer(data=request.data)
-        user = request.user
-        try:
-            if ser.is_valid(True):
-                details = ser.validated_data.get('details', {})
-                hostname = ser.validated_data.get('hostname', "").strip()
-                ip = ser.validated_data.get('ip', "")
-                id = ser.validated_data.get('id', "")
-                port = ser.validated_data.get('port', 80)
-                cluster_name = ser.validated_data.get('cluster_name', "").strip()
-                cluster_version = ser.validated_data.get('cluster_version', "")
-                priority = ser.validated_data.get('priority', 0)
-
-        except ValidationError as e:
-
-            return R.failure(data=e.detail)
-
-        config = self.create_agent_config(user, details, hostname, ip, port, cluster_name, cluster_version, priority, id)
-        if config:
-            return R.success(msg=_('保存成功'))
-        else:
-            return R.failure(msg=_('保存失败'))
-
-
-def intable_validate(value):
-    try:
-        a = int(value)
-    except ValueError as e:
-        raise serializers.ValidationError('This field must be an intable.')
-
-
-class AgentConfigSettingV2TargetSerializer(serializers.Serializer):
-    target_type = serializers.ChoiceField(TargetType.choices)
-    opt = serializers.ChoiceField(TargetOperator.choices)
-    value = serializers.CharField()
-
-
-class AgentConfigSettingV2MetricSerializer(serializers.Serializer):
-    metric_type = serializers.ChoiceField(MetricType.choices)
-    opt = serializers.ChoiceField(MetricOperator.choices)
-    value = serializers.CharField(validators=[intable_validate])
-
-
-class AgentConfigSettingV2Serializer(serializers.Serializer):
-    name = serializers.CharField()
-    targets = serializers.ListField(
-        child=AgentConfigSettingV2TargetSerializer())
-    metric_group = serializers.ChoiceField(MetricGroup.choices)
-    metrics = serializers.ListField(
-        child=AgentConfigSettingV2MetricSerializer())
-    interval = serializers.IntegerField(default=30)
-    deal = serializers.ChoiceField(DealType.choices)
-    is_enable = serializers.IntegerField()
-
-
-def get_priority_max_now() -> int:
-    res = IastCircuitConfig.objects.all().aggregate(Max("priority"))
-    return res["priority__max"] + 1
-
-
-def get_priority_min_now() -> int:
-    res = IastCircuitConfig.objects.all().aggregate(Min("priority"))
-    return res["priority__min"] - 1
-
-
-def config_create(data, user):
-    fields = ('name', 'metric_group', 'is_enable', 'deal',
-              "interval")
-    filted_data = get_data_from_dict_by_key(data, fields)
-    metric_types = get_metric_types(data['metrics'])
-    targets = get_targets(data['targets'])
-    obj = IastCircuitConfig.objects.create(**filted_data,
-                                           metric_types=metric_types,
-                                           target_types=targets,
-                                           priority=get_priority_max_now(),
-                                           user=user)
-    for i in data['targets']:
-        create_target(i, obj)
-
-    for i in data['metrics']:
-        create_metric(i, obj)
-
-
-def config_update(data, config_id):
-    fields = ('name', 'metric_group', 'is_enable', 'deal',
-              "interval")
-    filted_data = get_data_from_dict_by_key(data, fields)
-    metric_types = get_metric_types(data['metrics'])
-    targets = get_targets(data['targets'])
-    IastCircuitConfig.objects.filter(
-        pk=config_id).update(**filted_data,
-                             metric_types=metric_types,
-                             target_types=targets)
-    IastCircuitTarget.objects.filter(
-        circuit_config_id=config_id).delete()
-    IastCircuitMetric.objects.filter(
-        circuit_config_id=config_id).delete()
-    obj = IastCircuitConfig.objects.filter(pk=config_id).first()
-    if obj is None:
-        return
-    for i in data['targets']:
-        create_target(i, obj)
-
-    for i in data['metrics']:
-        create_metric(i, obj)
-
-
-def create_metric(metrics: dict | OrderedDict,
-                  circuit_config: IastCircuitConfig):
-    IastCircuitMetric.objects.create(circuit_config=circuit_config, **metrics)
-
-
-def create_target(target: dict | OrderedDict,
-                  circuit_config: IastCircuitConfig):
-    IastCircuitTarget.objects.create(circuit_config=circuit_config, **target)
-
-
-def get_metric_types(metrics):
-    str_list = []
-    for metric in metrics:
-        str_list.append(str(MetricType(metric['metric_type']).label))
-    return "、".join(str_list)
-
-
-def get_targets(targets):
-    str_list = []
-    for target in targets:
-        str_list.append(str(TargetType(target['target_type']).label))
-    res = "、".join(str_list)
-    if not res:
-        return str(_("全部"))
-    return res
-
-
-def get_data_from_dict_by_key(dic: ReturnDict | Dict,
-                              fields: Iterable) -> Dict:
-    return {i: dic[i] for i in fields}
-
-
-# when target_priority < config.priorty
-def set_config_change_lt(config_id, target_priority: int):
-    config = IastCircuitConfig.objects.filter(pk=config_id).first()
-    if not config:
-        return
-    IastCircuitConfig.objects.filter(
-        priority__gte=target_priority,
-        priority__lt=config.priority).update(priority=F('priority') + 1)
-    config.priority = target_priority
-    config.save()
-
-
-def set_config_top(config_id):
-    return set_config_change_lt(config_id,
-                                target_priority=get_priority_min_now())
-
-
-# when target_priority > config.priorty
-def set_config_change_gt(config_id, target_priority: int):
-    config = IastCircuitConfig.objects.filter(pk=config_id).first()
-    if not config:
-        return
-    IastCircuitConfig.objects.filter(
-        priority__lte=target_priority,
-        priority__gt=config.priority).update(priority=F('priority') - 1)
-    config.priority = target_priority
-    config.save()
-
-
-def set_config_bottom(config_id):
-    set_config_change_gt(config_id, target_priority=get_priority_max_now())
-
-
-def set_config_change_proprity(config_id, priority_range: list):
-    config = IastCircuitConfig.objects.filter(pk=config_id).first()
-    if not config:
-        return
-    if min(priority_range) > config.priority:
-        set_config_change_gt(config.id, min(priority_range))
-    if max(priority_range) < config.priority:
-        set_config_change_lt(config.id, max(priority_range))
-
-
-class AgentThresholdConfigV2(TalentAdminEndPoint, viewsets.ViewSet):
-    name = "api-v1-agent-threshold-config-setting-v2"
-    description = _("config Agent V2")
-
-    @extend_schema_with_envcheck(
-        [AgentConfigSettingV2Serializer],
-        summary=_('Create AgentThresholdConfig'),
-        description=_("Create AgentThresholdConfigV2"),
-        tags=[_('AgentThresholdConfigV2')])
-    def create(self, request):
-        ser = AgentConfigSettingV2Serializer(data=request.data)
-        try:
-            if ser.is_valid(True):
-                pass
-        except ValidationError as e:
-            return R.failure(data=e.detail)
-        config_create(ser.data, request.user)
-        return R.success()
-
-    @extend_schema_with_envcheck(
-        summary=_('AgentThresholdConfig Detail'),
-        tags=[_('AgentThresholdConfigV2')])
-    def retrieve(self, request, pk):
-        obj = IastCircuitConfig.objects.filter(pk=pk,
-                                               is_deleted=0).values().first()
-        if not obj:
-            return R.failure()
-        obj['targets'] = list(
-            IastCircuitTarget.objects.filter(
-                circuit_config_id=pk).values().all())
-        obj['metrics'] = list(
-            IastCircuitMetric.objects.filter(
-                circuit_config_id=pk).values().all())
-        return R.success(data=obj)
-
-    @extend_schema_with_envcheck(
-        summary=_('AgentThresholdConfig List'),
-        tags=[_('AgentThresholdConfigV2')])
-    def list(self, request):
-        #        page = request.query_params.get('page', 1)
-        #        page_size = request.query_params.get("page_size", 10)
-        queryset = IastCircuitConfig.objects.filter(
-            is_deleted=0).order_by('priority').prefetch_related(
-                'iastcircuittarget_set', 'iastcircuitmetric_set').all()
-        # page_summary, page_data = self.get_paginator(queryset, page, page_size)
-        obj_list = []
-        for data in queryset:
-            obj = model_to_dict(data)
-            obj['targets'] = list(data.iastcircuittarget_set.values().all())
-            obj['metrics'] = list(data.iastcircuitmetric_set.values().all())
-            obj_list.append(obj)
-        return R.success(data=obj_list)
-
-    @extend_schema_with_envcheck(
-        summary=_('Update AgentThresholdConfig'),
-        description=_("Update AgentThresholdConfigV2"),
-        tags=[_('AgentThresholdConfigV2')])
-    def update(self, request, pk):
-        ser = AgentConfigSettingV2Serializer(data=request.data)
-        try:
-            if ser.is_valid(True):
-                pass
-        except ValidationError as e:
-            return R.failure(data=e.detail)
-        config_update(ser.data, pk)
-        return R.success()
-
-    @extend_schema_with_envcheck(
-        summary=_('重置 AgentThresholdConfig'),
-        description=_("重置 AgentThresholdConfigV2"),
-        tags=[_('AgentThresholdConfigV2')])
-    def reset(self, request, pk):
-        if IastCircuitConfig.objects.filter(pk=pk).exists():
-            config = IastCircuitConfig.objects.filter(pk=pk, ).first()
-            if config is None:
-                return R.failure()
-            mg = MetricGroup(config.metric_group)
-            data = DEFAULT_CIRCUITCONFIG[mg.name]
-            config_update(data, pk)
-            return R.success()
-        return R.failure()
-
-    def change_priority(self, request, pk):
-        type_ = request.data.get('type')
-        priority_range = request.data.get('priority_range')
-        if IastCircuitConfig.objects.filter(pk=pk).exists():
-            if type_ == 1:
-                set_config_top(pk)
-                return R.success()
-            if type_ == 2 and priority_range:
-                set_config_change_proprity(pk, priority_range)
-                return R.success()
-            if type_ == 3:
-                set_config_bottom(pk)
-                return R.success()
-        return R.failure()
-
-    @extend_schema_with_envcheck(
-        summary=_('Delete AgentThresholdConfig'),
-        description=_("Delete AgentThresholdConfigV2"),
-        tags=[_('AgentThresholdConfigV2')])
-    def delete(self, request, pk):
-        IastCircuitConfig.objects.filter(pk=pk).update(is_deleted=1)
-        return R.success()
-
-    @extend_schema_with_envcheck(
-        summary=_('获取 AgentThresholdConfig 枚举'),
-        description=_("获取 AgentThresholdConfigV2 枚举"),
-        tags=[_('AgentThresholdConfigV2')])
-    def enum(self, request, enumname):
-        able_to_search = (TargetType, MetricType, MetricGroup, TargetOperator,
-                          MetricOperator, DealType, SystemMetricType,
-                          JVMMetricType, ApplicationMetricType)
-        able_to_search_dict = {
-            underscore(item.__name__): item
-            for item in able_to_search
-        }
-        if enumname not in able_to_search_dict.keys():
-            return R.failure()
-        return R.success(data=convert_choices_to_value_dict(
-            able_to_search_dict.get(enumname)))
-
-    @extend_schema_with_envcheck(
-        summary=_('获取 AgentThresholdConfig 所有枚举'),
-        tags=[_('AgentThresholdConfigV2')])
-    def enumall(self, request):
-        able_to_search = (TargetType, MetricType, MetricGroup, TargetOperator,
-                          MetricOperator, DealType, SystemMetricType,
-                          JVMMetricType, ApplicationMetricType)
-        res = {
-            underscore(item.__name__): convert_choices_to_value_dict(item)
-            for item in able_to_search
-        }
-        res['UNIT_DICT'] = UNIT_DICT
-        return R.success(data=res)
-
-
-def convert_choices_to_dict(choices):
-    fields = ['value', 'name', 'label']
-    return [{field: getattr(choice, field)
-             for field in fields}
-            for choice in choices]
-
-
-def convert_choices_to_value_dict(choices):
-    fields = ['name', 'label']
-    return {
-        choice.value: {field: getattr(choice, field)
-                       for field in fields}
-        for choice in choices
-    }
diff --git a/dongtai_web/threshold/del_threshold_setting.py b/dongtai_web/threshold/del_threshold_setting.py
deleted file mode 100644
index 1075ccc48..000000000
--- a/dongtai_web/threshold/del_threshold_setting.py
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/usr/bin/env python
-# -*- coding:utf-8 -*-
-# author:sjh
-# software: PyCharm
-# project: webApi
-# agent webHook setting
-import time
-
-from dongtai_common.endpoint import UserEndPoint, R
-from dongtai_common.models.agent_config import IastAgentConfig
-from django.utils.translation import gettext_lazy as _
-from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
-from dongtai_web.serializers.agent_config import AgentWebHookDelSerializer
-from rest_framework.serializers import ValidationError
-
-_ResponseSerializer = get_response_serializer(status_msg_keypair=(
-    ((201, _('The setting is complete')), ''),
-    ((202, _('Incomplete parameter, please try again later')), '')
-))
-
-
-class DelAgentThresholdConfig(UserEndPoint):
-    name = "api-v1-agent-Threshold-config-del"
-    description = _("del webHook Agent")
-
-    @extend_schema_with_envcheck(
-        tags=[_('Agent')],
-        summary=_('Agent webHook delete'),
-        description=_("Delete agent traffic reporting data forwarding address configuration"),
-        response_schema=_ResponseSerializer)
-    def post(self, request):
-        ser = AgentWebHookDelSerializer(data=request.data)
-        user = request.user
-        if ser.is_valid(False):
-            id = ser.validated_data.get('id', None)
-        else:
-            return R.failure(msg=_('Incomplete parameter, please check again'))
-        config = IastAgentConfig.objects.filter(user=user, id=id).delete()
-        if config:
-            return R.success(msg=_('Config has been deleted successfully'))
-        else:
-            R.failure(msg=_('Failed to delete config'))
diff --git a/dongtai_web/threshold/del_webhook_setting.py b/dongtai_web/threshold/del_webhook_setting.py
deleted file mode 100644
index 77f4f2f2b..000000000
--- a/dongtai_web/threshold/del_webhook_setting.py
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/usr/bin/env python
-# -*- coding:utf-8 -*-
-# author:sjh
-# software: PyCharm
-# project: webApi
-# agent webHook setting
-import time
-
-from dongtai_common.endpoint import UserEndPoint, R
-from dongtai_common.models.agent_webhook_setting import IastAgentUploadTypeUrl
-from django.utils.translation import gettext_lazy as _
-from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
-from dongtai_web.serializers.agent_config import AgentWebHookDelSerializer
-from rest_framework.serializers import ValidationError
-
-_ResponseSerializer = get_response_serializer(status_msg_keypair=(
-    ((201, _('The setting is complete')), ''),
-    ((202, _('Incomplete parameter, please try again later')), '')
-))
-
-
-class DelAgentWebHookConfig(UserEndPoint):
-    name = "api-v1-agent-webHook-config-del"
-    description = _("del webHook Agent")
-
-    @extend_schema_with_envcheck(
-        tags=[_('Agent')],
-        summary=_('Agent webHook delete'),
-        description=_("Delete agent traffic reporting data forwarding address configuration"),
-        response_schema=_ResponseSerializer)
-    def post(self, request):
-        ser = AgentWebHookDelSerializer(data=request.data)
-        user = request.user
-        if ser.is_valid(False):
-            id = ser.validated_data.get('id', None)
-        else:
-            return R.failure(msg=_('Incomplete parameter, please check again'))
-        config = IastAgentUploadTypeUrl.objects.filter(user=user, id=id).delete()
-        if config:
-            return R.success(msg=_('Config has been deleted successfully'))
-        else:
-            R.failure(msg=_('Failed to delete config'))
diff --git a/dongtai_web/threshold/get_config_setting.py b/dongtai_web/threshold/get_config_setting.py
deleted file mode 100644
index edf0ba002..000000000
--- a/dongtai_web/threshold/get_config_setting.py
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/usr/bin/env python
-# -*- coding:utf-8 -*-
-# author:sjh
-# software: PyCharm
-# project: webApi
-# agent threshold setting
-import time
-
-from django.forms import model_to_dict
-from dongtai_common.endpoint import UserEndPoint, R
-from dongtai_common.models.agent_config import IastAgentConfig
-from django.utils.translation import gettext_lazy as _
-from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
-
-_ResponseSerializer = get_response_serializer(status_msg_keypair=(
-    ((201, _('Get success')), ''),
-    ((202, _('Incomplete parameter, please try again later')), '')
-))
-
-
-class GetAgentThresholdConfig(UserEndPoint):
-    name = "api-v1-agent-threshold-config-get"
-    description = _("config Agent")
-
-    @extend_schema_with_envcheck(
-        tags=[_('Agent')],
-        summary=_('Agent threshold Config'),
-        description=_("Configure agent disaster recovery strategy"),
-        response_schema=_ResponseSerializer)
-    def get(self, request):
-        user = request.user
-        configData = IastAgentConfig.objects.filter(user=user)
-        result = []
-        if configData:
-            for item in configData:
-                data = model_to_dict(item)
-                data_detail = {}
-                if data['details']:
-                    data_detail = data['details']
-                del data['user']
-                del data['details']
-
-                if isinstance(data_detail, dict):
-
-                    data['enableAutoFallback'] = data_detail.get("enableAutoFallback", None)
-                    data['hookLimitTokenPerSecond'] = data_detail.get("hookLimitTokenPerSecond", None)
-                    data['heavyTrafficLimitTokenPerSecond'] = data_detail.get("heavyTrafficLimitTokenPerSecond", None)
-                    data['cpuUsagePercentage'] = data_detail.get("performanceLimitMaxThreshold", {}).get("cpuUsage", {}).get("cpuUsagePercentage", None)
-                    data['memUsagePercentage'] = data_detail.get("performanceLimitMaxThreshold", {}).get("memoryUsage", {}).get("memUsagePercentage", None)
-                else:
-                    data['enableAutoFallback'] = ""
-                    data['hookLimitTokenPerSecond'] = ""
-                    data['heavyTrafficLimitTokenPerSecond'] = ""
-                    data['cpuUsagePercentage'] = ""
-                    data['memUsagePercentage'] = ""
-                result.append(data)
-        else:
-            result = []
-        return R.success(msg=_('Successfully'), data={"result": result})
diff --git a/dongtai_web/threshold/get_config_setting_detail.py b/dongtai_web/threshold/get_config_setting_detail.py
deleted file mode 100644
index bba1a96a9..000000000
--- a/dongtai_web/threshold/get_config_setting_detail.py
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/usr/bin/env python
-# -*- coding:utf-8 -*-
-# author:sjh
-# software: PyCharm
-# project: webApi
-# agent threshold setting
-import time
-
-from django.forms import model_to_dict
-from dongtai_common.endpoint import UserEndPoint, R
-from dongtai_common.models.agent_config import IastAgentConfig
-from django.utils.translation import gettext_lazy as _
-from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
-
-_ResponseSerializer = get_response_serializer(status_msg_keypair=(
-    ((201, _('Get detail success')), ''),
-    ((202, _('Incomplete parameter, please try again later')), '')
-))
-
-
-class GetAgentThresholdConfigDetail(UserEndPoint):
-    name = "api-v1-agent-threshold-config-get-detail"
-    description = _("config Agent")
-
-    @extend_schema_with_envcheck(
-        tags=[_('Agent')],
-        summary=_('Agent threshold Config'),
-        description=_("Configure agent disaster recovery strategy"),
-        response_schema=_ResponseSerializer)
-    def get(self, request, pk):
-        user = request.user
-        configData = IastAgentConfig.objects.filter(user=user, pk=pk).first()
-        result = {}
-        if configData:
-            result = model_to_dict(configData)
-
-        return R.success(msg=_('Successfully'), data={"result": result})
diff --git a/dongtai_web/threshold/get_webhook_setting.py b/dongtai_web/threshold/get_webhook_setting.py
deleted file mode 100644
index e462bc3a6..000000000
--- a/dongtai_web/threshold/get_webhook_setting.py
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/usr/bin/env python
-# -*- coding:utf-8 -*-
-# author:sjh
-# software: PyCharm
-# project: webApi
-# agent threshold setting
-import time
-
-from django.forms import model_to_dict
-from dongtai_common.endpoint import UserEndPoint, R
-
-from dongtai_common.models.agent_webhook_setting import IastAgentUploadTypeUrl
-
-from django.utils.translation import gettext_lazy as _
-from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
-
-_ResponseSerializer = get_response_serializer(status_msg_keypair=(
-    ((201, _('Get success')), ''),
-    ((202, _('Incomplete parameter, please try again later')), '')
-))
-
-
-class GetAgentWebHookConfig(UserEndPoint):
-    name = "api-v1-agent-webHook-config-get"
-    description = _("config Agent")
-
-    @extend_schema_with_envcheck(
-        tags=[_('WebHook')],
-        summary=_('WebHook threshold Config get'),
-        description=_("WebHook threshold list"),
-        response_schema=_ResponseSerializer)
-    def get(self, request):
-        user = request.user
-        configData = IastAgentUploadTypeUrl.objects.filter(user=user).order_by("-create_time")
-        data = []
-        if configData:
-            for item in configData:
-                itemData = model_to_dict(item)
-                del itemData['user']
-                data.append(itemData)
-
-        return R.success(msg=_('Successfully'), data={"result": data})
diff --git a/dongtai_web/threshold/tests.py b/dongtai_web/threshold/tests.py
deleted file mode 100644
index cda931244..000000000
--- a/dongtai_web/threshold/tests.py
+++ /dev/null
@@ -1,18 +0,0 @@
-from django.test import TestCase
-from dongtai_web.threshold.config_setting import (
-    get_data_from_dict_by_key,
-    AgentConfigSettingV2TargetSerializer,
-)
-from rest_framework.utils.serializer_helpers import ReturnDict
-
-class TypingTestCase(TestCase):
-
-    def test_typing_in_get_data_from_dict_by_key(self, ):
-        get_data_from_dict_by_key({"213123132": "123123132"}, ("213123132", ))
-        ser = AgentConfigSettingV2TargetSerializer(data={
-            "target_type": 1,
-            "opt": 1,
-            "value": "21313"
-        })
-        ser.is_valid()
-        get_data_from_dict_by_key(ser.data, ("target_type", "opt", "value"))
diff --git a/dongtai_web/threshold/webhook_setting.py b/dongtai_web/threshold/webhook_setting.py
deleted file mode 100644
index b553670e4..000000000
--- a/dongtai_web/threshold/webhook_setting.py
+++ /dev/null
@@ -1,68 +0,0 @@
-#!/usr/bin/env python
-# -*- coding:utf-8 -*-
-# author:sjh
-# software: PyCharm
-# project: webApi
-# agent webHook setting
-import time
-
-from dongtai_common.endpoint import UserEndPoint, R
-from dongtai_common.models.agent_webhook_setting import IastAgentUploadTypeUrl
-from django.utils.translation import gettext_lazy as _
-from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
-from dongtai_web.serializers.agent_config import AgentWebHookSettingSerializer
-from rest_framework.serializers import ValidationError
-
-_ResponseSerializer = get_response_serializer(status_msg_keypair=(
-    ((201, _('The setting is complete')), ''),
-    ((202, _('Incomplete parameter, please try again later')), '')
-))
-
-
-class AgentWebHookConfig(UserEndPoint):
-    name = "api-v1-agent-webHook-config-setting"
-    description = _("config webHook Agent")
-
-    def create_webHook_config(self, user, type_id, url, headers, id):
-        try:
-            setting = {}
-            if id is not None:
-                setting = IastAgentUploadTypeUrl.objects.filter(user=user, id=id).first()
-            if setting:
-                setting.type_id = type_id
-                setting.url = url
-                setting.headers = headers
-            else:
-                timestamp = int(time.time())
-                setting = IastAgentUploadTypeUrl(
-                    user=user,
-                    type_id=type_id,
-                    url=url,
-                    headers=headers,
-                    create_time=timestamp
-                )
-            setting.save()
-            return setting
-        except Exception as e:
-            return None
-
-    @extend_schema_with_envcheck(
-        tags=[_('Agent')],
-        summary=_('Agent webHook Config'),
-        description=_("Agent traffic reporting data forwarding address configuration"),
-        response_schema=_ResponseSerializer)
-    def post(self, request):
-        ser = AgentWebHookSettingSerializer(data=request.data)
-        user = request.user
-        if ser.is_valid(False):
-            id = ser.validated_data.get('id', None)
-            type_id = ser.validated_data.get('type_id', None)
-            headers = ser.validated_data.get('headers', {})
-            url = ser.validated_data.get('url', "").strip()
-        else:
-            return R.failure(msg=_('Incomplete parameter, please check again'))
-        config = self.create_webHook_config(user, type_id, url, headers, id)
-        if config:
-            return R.success(msg=_('Config has been created successfully'), data={"id": config.id})
-        else:
-            R.failure(msg=_('Failed to create config'))
diff --git a/dongtai_web/threshold/webhook_type.py b/dongtai_web/threshold/webhook_type.py
deleted file mode 100644
index 2acfd1b68..000000000
--- a/dongtai_web/threshold/webhook_type.py
+++ /dev/null
@@ -1,75 +0,0 @@
-#!/usr/bin/env python
-# -*- coding:utf-8 -*-
-# author:sjh
-# software: PyCharm
-# project: webApi
-# agent webHook setting
-import time
-
-from dongtai_common.endpoint import UserEndPoint, R
-
-from django.utils.translation import gettext_lazy as _
-from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
-_ResponseSerializer = get_response_serializer(status_msg_keypair=(
-    ((201, _('The type is return')), ''),
-    ((202, _('Incomplete parameter, please try again later')), '')
-))
-
-
-class AgentWebHookTypeList(UserEndPoint):
-    name = "api-v1-agent-webHook-type-list"
-    description = _("get webhook all type ")
-
-    @extend_schema_with_envcheck(
-        tags=[_('WebHook')],
-        summary=_('Agent webHook type'),
-        description=_("type list of agent webHook"),
-        response_schema=_ResponseSerializer)
-    def get(self, request):
-        typeData = [
-            {
-                "key": "错误日志",
-                "value": 81
-            }, {
-                "key": "心跳",
-                "value": 1
-            }, {
-                "key": "低风险漏洞",
-                "value": 33
-            }, {
-                "key": "调用链",
-                "value": 36
-            }, {
-                "key": "SCA",
-                "value": 17
-            }, {
-                "key": "SCA批量",
-                "value": 18
-            }, {
-                "key": "ApiSiteMap",
-                "value": 97
-            }, {
-                "key": "硬编码",
-                "value": 37
-            }, {
-                "key": "高频hook限流",
-                "value": 65
-            }, {
-                "key": "高频请求限流",
-                "value": 66
-            }, {
-                "key": "性能监控降级",
-                "value": 67
-            }, {
-                "key": "异常降级",
-                "value": 68
-            }, {
-                "key": "监控线程异常",
-                "value": 69
-            }, {
-                "key": "触发二次降级",
-                "value": 70
-            }
-        ]
-
-        return R.success(msg=_('Get type list successfully'), data={"result": typeData})
diff --git a/dongtai_web/urls.py b/dongtai_web/urls.py
index b34e92aa5..f1ac59aba 100644
--- a/dongtai_web/urls.py
+++ b/dongtai_web/urls.py
@@ -19,9 +19,6 @@
 from rest_framework.urlpatterns import format_suffix_patterns
 
 from dongtai_web.base.update_project_version import UpdateProjectVersion
-from dongtai_web.threshold.del_threshold_setting import DelAgentThresholdConfig
-from dongtai_web.threshold.del_webhook_setting import DelAgentWebHookConfig
-from dongtai_web.threshold.get_config_setting_detail import GetAgentThresholdConfigDetail
 from dongtai_web.views.agent_delete import AgentDeleteEndPoint
 from dongtai_web.views.agent_deploy import AgentDeploy
 from dongtai_web.views.agent_install import AgentInstall
@@ -122,17 +119,10 @@
 from dongtai_web.views.details_id import (AgentListWithid, ProjectListWithid,
                                           ScaListWithid, VulsListWithid)
 from dongtai_web.views.vul_recheck_v2 import VulReCheckv2
-from dongtai_web.threshold.config_setting import AgentThresholdConfig
-from dongtai_web.threshold.webhook_setting import AgentWebHookConfig
-from dongtai_web.threshold.get_webhook_setting import GetAgentWebHookConfig
-from dongtai_web.threshold.webhook_type import AgentWebHookTypeList
-from dongtai_web.threshold.get_config_setting import GetAgentThresholdConfig
 
 from dongtai_web.aggregation.aggregation_del import DelVulMany
 from dongtai_web.aggregation.aggregation_project_del import DelVulProjectLevel
 
-from dongtai_web.threshold.config_setting import (
-    AgentThresholdConfigV2, )
 from dongtai_web.vul_log.vul_log_view import VulLogViewSet
 from dongtai_web.vul_recheck_payload.vul_recheck_payload import VulReCheckPayloadViewSet
 from dongtai_web.header_vul.base import HeaderVulViewSet
@@ -263,45 +253,13 @@
     path('vul/list/ids', VulsListWithid.as_view()),
     path('sca/list/ids', ScaListWithid.as_view()),
     path('project/list/ids', ProjectListWithid.as_view()),
-    # user settings disaster recovery strategy
-    path('threshold/settings', AgentThresholdConfig.as_view()),
-    # get user settings disaster recovery strategy GetAgentThresholdConfig
-    path('threshold/settings/get', GetAgentThresholdConfig.as_view()),
-    path('threshold/settings/get/',
-         GetAgentThresholdConfigDetail.as_view()),
-    path('threshold/settings/del', DelAgentThresholdConfig.as_view()),
-    # user webhook setting agent static report  forward
-    path('webhook/settings', AgentWebHookConfig.as_view()),
-    path('webhook/type/list', AgentWebHookTypeList.as_view()),
-    path('webhook/type/del', DelAgentWebHookConfig.as_view()),
 
     # get webHook setting
-    path('webhook/settings/get', GetAgentWebHookConfig.as_view()),
     path('agent/summary/', AgentSummary.as_view()),
 
     # vul list page of sca and common vul
     path('vul_list_delete', DelVulMany.as_view()),
     path('project_vul_delete', DelVulProjectLevel.as_view()),
-    path('circuit_config',
-         AgentThresholdConfigV2.as_view({
-             "post": "create",
-             "get": "list"
-         })),
-    path('circuit_config/enum/all',
-         AgentThresholdConfigV2.as_view({"get": "enumall"})),
-    path('circuit_config//priority',
-         AgentThresholdConfigV2.as_view({"put": "change_priority"})),
-    path('circuit_config//reset',
-         AgentThresholdConfigV2.as_view({"put": "reset"})),
-    path('circuit_config/enum/',
-         AgentThresholdConfigV2.as_view({"get": "enum"})),
-    path(
-        'circuit_config/',
-        AgentThresholdConfigV2.as_view({
-            "put": "update",
-            "delete": "delete",
-            "get": "retrieve"
-        })),
     path("vullog/", VulLogViewSet.as_view({"get": "list"})),
     path(
         'vul_recheck_payload/',

From 2a690041df14ad46d6665b944b48e2eb763ce3b2 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Fri, 7 Jul 2023 17:04:16 +0800
Subject: [PATCH 076/161] feat: remove vul recheck api

---
 dongtai_web/urls.py                           |  21 --
 dongtai_web/views/vul_recheck.py              | 251 -----------------
 dongtai_web/views/vul_recheck_v2.py           | 258 ------------------
 dongtai_web/vul_recheck_payload/__init__.py   |   0
 .../vul_recheck_payload.py                    | 172 ------------
 5 files changed, 702 deletions(-)
 delete mode 100644 dongtai_web/views/vul_recheck.py
 delete mode 100644 dongtai_web/views/vul_recheck_v2.py
 delete mode 100644 dongtai_web/vul_recheck_payload/__init__.py
 delete mode 100644 dongtai_web/vul_recheck_payload/vul_recheck_payload.py

diff --git a/dongtai_web/urls.py b/dongtai_web/urls.py
index f1ac59aba..7b4b7868f 100644
--- a/dongtai_web/urls.py
+++ b/dongtai_web/urls.py
@@ -89,7 +89,6 @@
     VulDetailV2,
 )
 from dongtai_web.views.vul_list_for_plugin import VulListEndPoint
-from dongtai_web.views.vul_recheck import VulReCheck
 from dongtai_web.views.vul_request_replay import RequestReplayEndPoint
 from dongtai_web.views.vul_status import VulStatus
 from dongtai_web.views.vul_summary import VulSummary
@@ -118,13 +117,11 @@
 )
 from dongtai_web.views.details_id import (AgentListWithid, ProjectListWithid,
                                           ScaListWithid, VulsListWithid)
-from dongtai_web.views.vul_recheck_v2 import VulReCheckv2
 
 from dongtai_web.aggregation.aggregation_del import DelVulMany
 from dongtai_web.aggregation.aggregation_project_del import DelVulProjectLevel
 
 from dongtai_web.vul_log.vul_log_view import VulLogViewSet
-from dongtai_web.vul_recheck_payload.vul_recheck_payload import VulReCheckPayloadViewSet
 from dongtai_web.header_vul.base import HeaderVulViewSet
 from dongtai_web.views.new_project_query import NewProjectVersionList
 from dongtai_web.enum.hook_rules import HookRuleEnumEndPoint
@@ -163,7 +160,6 @@
     path('vuln/', VulDetail.as_view()),
     path('vuln/status', VulStatus.as_view()),
     path('vuln/delete/', VulDelete.as_view()),
-    path('vul/recheck', VulReCheck.as_view()),
     path('vul/status_list', VulnerabilityStatusView.as_view()),
     path('plugin/vuln/list', VulListEndPoint.as_view()),
     path('plugin/vuln/count', VulCountForPluginEndPoint.as_view()),
@@ -261,22 +257,6 @@
     path('vul_list_delete', DelVulMany.as_view()),
     path('project_vul_delete', DelVulProjectLevel.as_view()),
     path("vullog/", VulLogViewSet.as_view({"get": "list"})),
-    path(
-        'vul_recheck_payload/',
-        VulReCheckPayloadViewSet.as_view({
-            'get': "retrieve",
-            'put': 'update',
-            'delete': 'delete'
-        })),
-    path('vul_recheck_payload',
-         VulReCheckPayloadViewSet.as_view({
-             'get': "list",
-             'post': "create",
-         })),
-    path('vul_recheck_payload/status',
-         VulReCheckPayloadViewSet.as_view({
-             'put': "status_change",
-         })),
     path('header_vul', HeaderVulViewSet.as_view({
         'get': "list",
     })),
@@ -302,7 +282,6 @@
 
 urlpatterns = [path('api/v1/', include(urlpatterns))]
 urlpatterns.extend([
-    path('api/v2/vul/recheck', VulReCheckv2.as_view()),
     path('api/v2/vuln/', VulDetailV2.as_view()),
     path('api/v2/agents', AgentListv2.as_view({"get": "pagenation_list"})),
     path('api/v2/agents/summary', AgentListv2.as_view({"get": "summary"})),
diff --git a/dongtai_web/views/vul_recheck.py b/dongtai_web/views/vul_recheck.py
deleted file mode 100644
index 0197415d4..000000000
--- a/dongtai_web/views/vul_recheck.py
+++ /dev/null
@@ -1,251 +0,0 @@
-#!/usr/bin/env python
-# -*- coding:utf-8 -*-
-# author:owefsad
-# software: PyCharm
-# project: lingzhi-webapi
-import logging
-
-import time
-from dongtai_common.models.agent import IastAgent
-from dongtai_common.models.project import IastProject
-from dongtai_common.models.replay_queue import IastReplayQueue
-from dongtai_common.models.vulnerablity import IastVulnerabilityModel
-from dongtai_common.utils.validate import Validate
-from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
-from rest_framework import serializers
-
-from dongtai_common.endpoint import R
-from dongtai_common.utils import const
-from dongtai_common.endpoint import UserEndPoint
-from django.utils.translation import gettext_lazy as _
-from django.db.models import F
-from django.db.models import Q
-import threading
-
-from dongtai_web.vul_log.vul_log import log_recheck_vul
-
-logger = logging.getLogger('dongtai-webapi')
-
-
-class VulReCheckDataSerializer(serializers.Serializer):
-    no_agent = serializers.BooleanField(
-        help_text=_('Whether the project does not exist agent'))
-    pending = serializers.IntegerField(
-        help_text=_('Waiting queue length for replay'))
-    recheck = serializers.IntegerField(
-        help_text=_('Success queue length for replay'))
-    checking = serializers.IntegerField(
-        help_text=_('Checking queue length for replay'))
-
-
-_ResponseGetSerializer = get_response_serializer(
-    VulReCheckDataSerializer(),
-    status_msg_keypair=(
-        ((201, _('Handle success')), ''),
-        ((202, _('Item ID should not be empty')), ''),
-        ((202, _('Incorrect format parameter')), ''),
-        ((202, _('Batch playback error')), ''),
-        ((202, _('Current application has not been associated with probes and cannot be reproduced.')), ''),
-        ((202, _('No permission to access')), ''),
-    ))
-_ResponsePostSerializer = get_response_serializer(
-    VulReCheckDataSerializer(),
-    status_msg_keypair=(
-        ((201, _('Handle success')), ''),
-        ((202, _('IDS should not be empty')), ''),
-        ((202, _('IDS must be: Vulnerability ID, Vulnerability ID Format')), ''),
-        ((202, _('Vulnerability replay error')), ''),
-    ))
-
-
-class VulReCheck(UserEndPoint):
-    @staticmethod
-    def recheck(vul_queryset):
-        timestamp = int(time.time())
-        waiting_count = 0
-        success_count = 0
-        re_success_count = 0
-        opt_vul_queryset = vul_queryset.only('agent__id', 'id')
-        vul_ids = [i.id for i in opt_vul_queryset]
-        vul_id_agentmap = {i.id: i.agent_id for i in opt_vul_queryset}
-        history_replay_vul_ids = IastReplayQueue.objects.filter(
-            relation_id__in=vul_ids,
-            replay_type=const.VUL_REPLAY).order_by('relation_id').values_list(
-                'relation_id', flat=True).distinct()
-        waiting_count = IastReplayQueue.objects.filter(
-            Q(relation_id__in=vul_ids)
-            & Q(replay_type=const.VUL_REPLAY)
-            & Q(state__in=(const.PENDING, const.WAITING))).count()
-        re_success_count = IastReplayQueue.objects.filter(
-            Q(relation_id__in=[i.id for i in opt_vul_queryset])
-            & Q(replay_type=const.VUL_REPLAY)
-            & ~Q(state__in=(const.PENDING, const.WAITING))).update(
-                state=const.WAITING,
-                count=F('count') + 1,
-                update_time=timestamp)
-        vuls_not_exist = set(vul_ids) - set(history_replay_vul_ids)
-        success_count = len(vuls_not_exist)
-        IastReplayQueue.objects.bulk_create(
-            [
-                IastReplayQueue(agent_id=vul_id_agentmap[vul_id],
-                                relation_id=vul_id,
-                                state=const.WAITING,
-                                count=1,
-                                create_time=timestamp,
-                                update_time=timestamp,
-                                replay_type=const.VUL_REPLAY)
-                for vul_id in vuls_not_exist
-            ],
-            ignore_conflicts=True)
-        vul_queryset.update(status_id=1, latest_time=timestamp)
-        return waiting_count, success_count, re_success_count
-
-    @staticmethod
-    def vul_check_for_queryset(vul_queryset):
-        active_agent_ids = IastAgent.objects.filter(
-            id__in=vul_queryset.values('agent_id'),
-            online=const.RUNNING,
-            is_core_running=const.CORE_IS_RUNNING).values(
-                "id").distinct().all()
-        no_agent = vul_queryset.filter(~Q(
-            agent_id__in=active_agent_ids)).count()
-        waiting_count, success_count, re_success_count = VulReCheck.recheck(
-            vul_queryset)
-        return no_agent, waiting_count, success_count, re_success_count
-
-    @extend_schema_with_envcheck(
-        [{
-            'name':
-            'type',
-            'type':
-            str,
-            'description':
-            _('''available options are ("all","project").
-                Corresponding to all or specific project respectively.''')
-        }, {
-            'name':
-            "projectId",
-            'type':
-            int,
-            'description':
-            _("""The corresponding id of the Project.
-            Only If the type is project, the projectId here will be used.""")
-        }],
-        tags=[_('Vulnerability')],
-        summary=_("Vulnerability verification"),
-        description=_("""Verify the user's corresponding vulnerabilities.
-            Need to specify the type"""),
-        response_schema=_ResponsePostSerializer
-    )
-    def post(self, request):
-        """
-        :param request:
-        :return:
-        """
-        try:
-            vul_ids = request.data.get('ids')
-            if vul_ids is None or vul_ids == '':
-                return R.failure(_("IDS should not be empty"))
-
-            vul_ids = vul_ids.split(',')
-            if Validate.is_number(vul_ids) is False:
-                return R.failure(_('IDS must be: Vulnerability ID, Vulnerability ID Format'))
-
-            auth_agents = self.get_auth_agents_with_user(user=request.user)
-            vul_queryset = IastVulnerabilityModel.objects.filter(
-                id__in=vul_ids, agent__in=auth_agents)
-            no_agent, waiting_count, success_count, re_success_count = self.vul_check_for_queryset(
-                vul_queryset)
-            # 加重放日志 vul_ids
-            log_recheck_vul(request.user.id, request.user.username, vul_ids, "待验证")
-
-            # def log_recheck_vul(user_id: int, user_name: str, vul_id: list,  vul_status: str):
-            return R.success(data={
-                "no_agent": no_agent,
-                "pending": waiting_count,
-                "recheck": re_success_count,
-                "checking": success_count
-            }, msg=_('Handle success'))
-
-        except Exception as e:
-            logger.error(f' msg:{e}')
-            return R.failure(msg=_('Vulnerability replay error'))
-
-    def vul_check_for_project(self, project_id, auth_users):
-        try:
-            project_exist = IastProject.objects.values("id").filter(
-                id=project_id, user__in=auth_users).exists()
-            if project_exist:
-                agent_queryset = IastAgent.objects.values("id").filter(
-                    bind_project_id=project_id)
-                if agent_queryset:
-                    agent_ids = agent_queryset.values_list('id', flat=True)
-                    vul_queryset = IastVulnerabilityModel.objects.filter(
-                        agent_id__in=agent_ids)
-                    waiting_count, success_count, re_success_count = self.recheck(
-                        vul_queryset)
-                    return True, waiting_count, re_success_count, success_count, None
-                else:
-                    return False, 0, 0, 0, _(
-                        'Current application has not been associated with probes and cannot be reproduced.'
-                    )
-            else:
-                return False, 0, 0, 0, _('No permission to access')
-        except Exception as e:
-            logger.error(f' msg:{e}', exc_info=True)
-            return False, 0, 0, 0, _('Batch playback error')
-
-    @extend_schema_with_envcheck(
-        [{
-            'name':
-            'type',
-            'type':
-            str,
-            'description':
-            _('''available options are ("all","project").
-                Corresponding to all or specific project respectively.''')
-        }, {
-            'name':
-            "projectId",
-            'type':
-            int,
-            'description':
-            _("""The corresponding id of the Project.
-            Only If the type is project, the projectId here will be used.""")
-        }],
-        tags=[_('Vulnerability')],
-        summary=_("Vulnerability verification"),
-        description=_("""Verify the user's corresponding vulnerabilities.
-            Need to specify the type"""),
-        response_schema=_ResponsePostSerializer
-    )
-    def get(self, request):
-
-        try:
-            check_type = request.query_params.get('type')
-            project_id = request.query_params.get('projectId')
-            if check_type == 'project' and not project_id:
-                return R.failure(msg=_("Item ID should not be empty"))
-            if check_type == 'all':
-                vul_queryset = IastVulnerabilityModel.objects.filter(
-                    agent__in=self.get_auth_agents_with_user(request.user))
-                no_agent, pending, recheck, checking = self.vul_check_for_queryset(
-                    vul_queryset)
-
-            elif check_type == 'project':
-                auth_users = self.get_auth_users(request.user)
-                status, pending, recheck, checking, msg = self.vul_check_for_project(
-                    project_id, auth_users=auth_users)
-                no_agent = None
-            return R.success(data={
-                "no_agent": no_agent if no_agent else 0,
-                "pending": pending,
-                "recheck": recheck,
-                "checking": checking
-            },
-                msg=_("Handle success"))
-            return R.failure(msg=_("Incorrect format parameter"))
-
-        except Exception as e:
-            logger.error(f'user_id:{request.user.id} msg:{e}', exc_info=True)
-            return R.failure(msg=_('Batch playback error'))
diff --git a/dongtai_web/views/vul_recheck_v2.py b/dongtai_web/views/vul_recheck_v2.py
deleted file mode 100644
index 1b455cd76..000000000
--- a/dongtai_web/views/vul_recheck_v2.py
+++ /dev/null
@@ -1,258 +0,0 @@
-#!/usr/bin/env python
-# -*- coding:utf-8 -*-
-# author:owefsad
-# software: PyCharm
-# project: lingzhi-webapi
-import logging
-
-import time
-from dongtai_common.models.agent import IastAgent
-from dongtai_common.models.project import IastProject
-from dongtai_common.models.replay_queue import IastReplayQueue
-from dongtai_common.models.vulnerablity import IastVulnerabilityModel
-from dongtai_web.aggregation.aggregation_common import turnIntListOfStr
-from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
-
-from dongtai_common.endpoint import R
-from dongtai_common.utils import const
-from dongtai_common.endpoint import UserEndPoint
-from django.utils.translation import gettext_lazy as _
-from django.db.models import F
-from django.db.models import Q
-import threading
-from dongtai_common.models.vul_recheck_payload import IastVulRecheckPayload
-
-logger = logging.getLogger('dongtai-webapi')
-
-_ResponseGetSerializer = get_response_serializer(
-    status_msg_keypair=(
-        ((201, _('Handle success')), ''),
-        ((202, _('Item ID should not be empty')), ''),
-        ((202, _('Incorrect format parameter')), ''),
-        ((202, _('Batch playback error')), ''),
-        ((202, _('Current application has not been associated with probes and cannot be reproduced.')), ''),
-        ((202, _('No permission to access')), ''),
-    ))
-_ResponsePostSerializer = get_response_serializer(
-    status_msg_keypair=(
-        ((201, _('Handle success')), ''),
-        ((202, _('IDS should not be empty')), ''),
-        ((202, _('IDS must be: Vulnerability ID, Vulnerability ID Format')), ''),
-        ((202, _('Vulnerability replay error')), ''),
-    ))
-
-
-class VulReCheckv2(UserEndPoint):
-    @staticmethod
-    def recheck(vul_queryset):
-        timestamp = int(time.time())
-        waiting_count = 0
-        success_count = 0
-        re_success_count = 0
-        opt_vul_queryset = vul_queryset.only('agent__id', 'id')
-        vul_ids = [i.id for i in opt_vul_queryset]
-        vul_id_agentmap = {i.id: i.agent_id for i in opt_vul_queryset}
-        history_replay_vul_ids = IastReplayQueue.objects.filter(
-            relation_id__in=vul_ids,
-            replay_type=const.VUL_REPLAY).order_by('relation_id').values_list(
-            'relation_id', flat=True).distinct()
-
-        waiting_count = IastReplayQueue.objects.filter(
-            Q(relation_id__in=vul_ids)
-            & Q(replay_type=const.VUL_REPLAY)
-            & Q(state__in=(const.PENDING, const.WAITING))).count()
-        re_success_count = IastReplayQueue.objects.filter(
-            Q(relation_id__in=[i.id for i in opt_vul_queryset])
-            & Q(replay_type=const.VUL_REPLAY)
-            & ~Q(state__in=(const.PENDING, const.WAITING))).update(
-            state=const.WAITING,
-            count=F('count') + 1,
-            update_time=timestamp)
-        vuls_not_exist = set(vul_ids)  # - set(history_replay_vul_ids)
-        success_count = len(vuls_not_exist)
-        vul_payload_dict = {}
-        for vul_id in vuls_not_exist:
-            vul_payload_dict[vul_id] = IastVulRecheckPayload.objects.filter(
-                strategy__iastvulnerabilitymodel__id=vul_id).values_list(
-                'pk', flat=True).all()
-        replay_queue = []
-        for key, value in vul_payload_dict.items():
-            item = [
-                IastReplayQueue(agent_id=vul_id_agentmap[key],
-                                relation_id=key,
-                                state=const.WAITING,
-                                count=1,
-                                create_time=timestamp,
-                                update_time=timestamp,
-                                replay_type=const.VUL_REPLAY,
-                                payload_id=payload_id) for payload_id in value
-            ]
-            if not item:
-                item = [
-                    IastReplayQueue(agent_id=vul_id_agentmap[key],
-                                    relation_id=key,
-                                    state=const.WAITING,
-                                    count=1,
-                                    create_time=timestamp,
-                                    update_time=timestamp,
-                                    replay_type=const.VUL_REPLAY)
-                ]
-            replay_queue += item
-        IastReplayQueue.objects.bulk_create(replay_queue,
-                                            ignore_conflicts=True)
-
-        for vul in vul_queryset:
-            vul.status_id = 1
-            vul.latest_time = timestamp
-            vul.save()
-
-        return waiting_count, success_count, re_success_count
-
-    @staticmethod
-    def vul_check_for_queryset(vul_queryset):
-        active_agent_ids = IastAgent.objects.filter(
-            id__in=vul_queryset.values('agent_id'),
-            online=const.RUNNING,
-            is_core_running=const.CORE_IS_RUNNING).values(
-            "id").distinct().all()
-        no_agent = vul_queryset.filter(~Q(
-            agent_id__in=active_agent_ids)).count()
-        waiting_count, success_count, re_success_count = VulReCheckv2.recheck(
-            vul_queryset)
-        return no_agent, waiting_count, success_count, re_success_count
-
-    @extend_schema_with_envcheck(
-        [{
-            'name':
-                'type',
-            'type':
-                str,
-            'description':
-                _('''available options are ("all","project").
-                Corresponding to all or specific project respectively.''')
-        }, {
-            'name':
-                "projectId",
-            'type':
-                int,
-            'description':
-                _("""The corresponding id of the Project.
-            Only If the type is project, the projectId here will be used.""")
-        }],
-        tags=[_('Vulnerability')],
-        summary=_("Vulnerability verification"),
-        description=_("""Verify the user's corresponding vulnerabilities.
-            Need to specify the type"""),
-        response_schema=_ResponsePostSerializer
-    )
-    def post(self, request):
-        """
-        :param request:
-        :return:
-        """
-        try:
-            vul_ids = request.data.get("ids", "")
-            department = request.user.get_relative_department()
-
-            queryset = IastVulnerabilityModel.objects.filter(
-                Q(is_del=0, project__department__in=department)
-                & ~Q(status_id__in=(3, 5, 6)))
-            ids_list = turnIntListOfStr(vul_ids)
-
-            vul_queryset = queryset.filter(id__in=ids_list)
-            if not vul_queryset.exists():
-                R.failure(msg="漏洞处于最终状态时不允许重新验证")
-            no_agent, waiting_count, success_count, re_success_count = self.vul_check_for_queryset(
-                vul_queryset)
-
-            return R.success(data={
-                "no_agent": no_agent,
-                "pending": waiting_count,
-                "recheck": re_success_count,
-                "checking": success_count
-            }, msg=_('Handle success'))
-
-        except Exception as e:
-            logger.error(f' msg:{e}')
-            return R.failure(msg=_('Vulnerability replay error'))
-
-    def vul_check_for_project(self, project_id, auth_users):
-        try:
-            project_exist = IastProject.objects.values("id").filter(
-                id=project_id, user__in=auth_users).exists()
-            if project_exist:
-                agent_queryset = IastAgent.objects.values("id").filter(
-                    bind_project_id=project_id)
-                if agent_queryset:
-                    agent_ids = agent_queryset.values_list('id', flat=True)
-                    vul_queryset = IastVulnerabilityModel.objects.filter(
-                        agent_id__in=agent_ids)
-                    waiting_count, success_count, re_success_count = self.recheck(
-                        vul_queryset)
-                    return True, waiting_count, re_success_count, success_count, None
-                else:
-                    return False, 0, 0, 0, _(
-                        'Current application has not been associated with probes and cannot be reproduced.'
-                    )
-            else:
-                return False, 0, 0, 0, _('No permission to access')
-        except Exception as e:
-            logger.error(f' msg:{e}', exc_info=True)
-            return False, 0, 0, 0, _('Batch playback error')
-
-    @extend_schema_with_envcheck(
-        [{
-            'name':
-                'type',
-            'type':
-                str,
-            'description':
-                _('''available options are ("all","project").
-                Corresponding to all or specific project respectively.''')
-        }, {
-            'name':
-                "projectId",
-            'type':
-                int,
-            'description':
-                _("""The corresponding id of the Project.
-            Only If the type is project, the projectId here will be used.""")
-        }],
-        tags=[_('Vulnerability')],
-        summary=_("Vulnerability verification"),
-        description=_("""Verify the user's corresponding vulnerabilities.
-            Need to specify the type"""),
-        response_schema=_ResponsePostSerializer
-    )
-    def get(self, request):
-
-        try:
-            check_type = request.query_params.get('type')
-            project_id = request.query_params.get('projectId')
-            if check_type == 'project' and not project_id:
-                return R.failure(msg=_("Item ID should not be empty"))
-            if check_type == 'all':
-                vul_queryset = IastVulnerabilityModel.objects.filter(
-                    agent__in=self.get_auth_agents_with_user(request.user))
-
-                def vul_check_thread():
-                    self.vul_check_for_queryset(vul_queryset)
-
-                t1 = threading.Thread(target=vul_check_thread, daemon=True)
-                t1.start()
-                return R.success(msg=_('Verification in progress'))
-            elif check_type == 'project':
-                auth_users = self.get_auth_users(request.user)
-
-                def vul_check_thread():
-                    self.vul_check_for_project(project_id,
-                                               auth_users=auth_users)
-
-                t1 = threading.Thread(target=vul_check_thread, daemon=True)
-                t1.start()
-                return R.success(msg=_("Verification in progress"))
-            return R.failure(msg=_("Incorrect format parameter"))
-
-        except Exception as e:
-            logger.error(f'user_id:{request.user.id} msg:{e}', exc_info=True)
-            return R.failure(msg=_('Batch playback error'))
diff --git a/dongtai_web/vul_recheck_payload/__init__.py b/dongtai_web/vul_recheck_payload/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/dongtai_web/vul_recheck_payload/vul_recheck_payload.py b/dongtai_web/vul_recheck_payload/vul_recheck_payload.py
deleted file mode 100644
index d3f555437..000000000
--- a/dongtai_web/vul_recheck_payload/vul_recheck_payload.py
+++ /dev/null
@@ -1,172 +0,0 @@
-import time
-
-from dongtai_common.endpoint import UserEndPoint, R, TalentAdminEndPoint
-from dongtai_common.models.agent_config import IastAgentConfig
-from django.utils.translation import gettext_lazy as _
-from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
-from dongtai_web.serializers.agent_config import AgentConfigSettingSerializer
-from rest_framework.serializers import ValidationError
-from rest_framework import viewsets
-from dongtai_common.models.vul_recheck_payload import IastVulRecheckPayload
-from rest_framework import serializers
-from django.db.models import Q
-from django.forms.models import model_to_dict
-from django.utils.translation import gettext as _
-from drf_spectacular.utils import extend_schema
-
-def get_or_none(classmodel, function, **kwargs):
-    try:
-        if function:
-            return function(classmodel.objects).get(**kwargs)
-        return classmodel.objects.get(**kwargs)
-    except classmodel.DoesNotExist:
-        return None
-
-
-class IastVulRecheckPayloadSerializer(serializers.Serializer):
-    value = serializers.CharField()
-    status = serializers.IntegerField()
-    strategy_id = serializers.IntegerField()
-    language_id = serializers.IntegerField()
-
-
-class IastVulRecheckPayloadListSerializer(serializers.Serializer):
-    keyword = serializers.CharField(required=False, default=None)
-    page = serializers.IntegerField()
-    page_size = serializers.IntegerField()
-    strategy_id = serializers.IntegerField(required=False, default=None)
-    language_id = serializers.IntegerField(required=False, default=None)
-
-
-def vul_recheck_payload_create(data, user_id):
-    IastVulRecheckPayload.objects.create(strategy_id=data['strategy_id'],
-                                         user_id=user_id,
-                                         value=data['value'],
-                                         status=data['status'],
-                                         language_id=data['language_id'])
-
-
-class AgentConfigSettingBatchV2Serializer(serializers.Serializer):
-    ids = serializers.ListField(child=serializers.IntegerField())
-
-
-def payload_update(pk, data):
-    IastVulRecheckPayload.objects.filter(pk=pk).update(**data)
-
-
-class VulReCheckPayloadViewSet(UserEndPoint, viewsets.ViewSet):
-    name = "api-v1-vul-recheck-payload"
-    description = _("config recheck payload V2")
-
-    @extend_schema(
-        summary="漏洞验证",
-        tags=[_("Vulnerability")],
-    )
-    def create(self, request):
-        ser = IastVulRecheckPayloadSerializer(data=request.data)
-        try:
-            if ser.is_valid(True):
-                pass
-        except ValidationError as e:
-            return R.failure(data=e.detail)
-        vul_recheck_payload_create(ser.data, request.user.id)
-        return R.success()
-
-    @extend_schema(
-        summary="漏洞验证",
-        tags=[_("Vulnerability")],
-    )
-    def retrieve(self, request, pk):
-        obj = get_or_none(
-            IastVulRecheckPayload,
-            lambda x: x.values('id', 'user__username', 'strategy__vul_name',
-                               'value', 'user_id', 'strategy_id', 'status',
-                               'create_time', 'language_id'),
-            pk=pk, user_id=request.user.id)
-        if obj is None:
-            return R.failure()
-        return R.success(data=obj)
-
-    @extend_schema(
-        summary="漏洞验证列表",
-        tags=[_("Vulnerability")],
-    )
-    def list(self, request):
-        ser = IastVulRecheckPayloadListSerializer(data=request.query_params)
-        try:
-            if ser.is_valid(True):
-                pass
-        except ValidationError as e:
-            return R.failure(data=e.detail)
-        q = Q(user_id=request.user.id) & ~Q(status=-1)
-        keyword = ser.data['keyword']
-        strategy_id = ser.data['strategy_id']
-        language_id = ser.data['language_id']
-        page = ser.data['page']
-        page_size = ser.data['page_size']
-        if keyword:
-            q = q & Q(value__icontains=keyword)
-        if strategy_id:
-            q = q & Q(strategy_id=strategy_id)
-        if language_id:
-            q = q & Q(language_id=language_id)
-        queryset = IastVulRecheckPayload.objects.filter(q).order_by(
-            '-create_time').values('id', 'user__username',
-                                   'strategy__vul_name', 'value', 'user_id',
-                                   'strategy_id', 'status', 'create_time',
-                                   'language_id')
-        page_summary, page_data = self.get_paginator(queryset, page, page_size)
-        return R.success(page=page_summary,
-                         data=list(page_data))
-
-    @extend_schema(
-        summary="漏洞验证",
-        tags=[_("Vulnerability")],
-    )
-    def update(self, request, pk):
-        ser = IastVulRecheckPayloadSerializer(data=request.data)
-        try:
-            if ser.is_valid(True):
-                pass
-        except ValidationError as e:
-            return R.failure(data=e.detail)
-        if IastVulRecheckPayload.objects.filter(
-                pk=pk, user_id=request.user.id).exists():
-            payload_update(pk, ser.data)
-            return R.success()
-        return R.success()
-
-    @extend_schema(
-        summary="漏洞验证",
-        tags=[_("Vulnerability")],
-    )
-    def delete(self, request, pk):
-        if IastVulRecheckPayload.objects.filter(
-                pk=pk, user_id=request.user.id).exists():
-            IastVulRecheckPayload.objects.filter(pk=pk).update(status=-1)
-            return R.success()
-        return R.failure()
-
-    @extend_schema(
-        summary="漏洞验证状态",
-        tags=[_("Vulnerability")],
-    )
-    def status_change(self, request):
-        mode = request.data.get('mode', 1)
-        q = ~Q(status=-1) & Q(user_id=request.user.id)
-        if mode == 1:
-            ids = request.data.get('ids', [])
-            status = request.data.get('status', 0)
-            q = q & Q(pk__in=ids)
-            IastVulRecheckPayload.objects.filter(q).update(status=status)
-        elif mode == 2:
-            status = request.data.get('status', 0)
-            q = q
-            IastVulRecheckPayload.objects.filter(q).update(status=status)
-        return R.success()
-
-    def status_all(self, request):
-        status = request.data.get('status', 0)
-        q = ~Q(status=-1)
-        IastVulRecheckPayload.objects.filter(q).update(status=status)
-        return R.success()

From 8046d534a079048b56085ae5f9243b997cadee39 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Mon, 10 Jul 2023 11:31:29 +0800
Subject: [PATCH 077/161] fix: type hint error

---
 dongtai_common/utils/init_schema.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/dongtai_common/utils/init_schema.py b/dongtai_common/utils/init_schema.py
index cd0d71b8c..87d487515 100644
--- a/dongtai_common/utils/init_schema.py
+++ b/dongtai_common/utils/init_schema.py
@@ -7,7 +7,7 @@
 import inspect
 
 logger = logging.getLogger("django")
-VIEW_CLASS_TO_SCHEMA: dict[type, dict[str, tuple[str, str, dict | None]]] = {}
+VIEW_CLASS_TO_SCHEMA: dict[type, dict[str, tuple[str, str, dict | None, str]]] = {}
 
 
 def init_schema() -> None:
@@ -23,6 +23,7 @@ def init_schema() -> None:
     if not path_prefix.startswith("^"):
         path_prefix = "^" + path_prefix  # make sure regex only matches from the start
     from dongtai_common.endpoint import EndPoint
+
     for path, path_regex, method, view in endpoints:
         if not issubclass(view.__class__, EndPoint):
             continue
@@ -40,4 +41,6 @@ def init_schema() -> None:
                     filepath,
                 )
         except Exception as e:
-            logger.error(f"unable to get schema: view {view} of path {path}", exc_info=e)
+            logger.error(
+                f"unable to get schema: view {view} of path {path}", exc_info=e
+            )

From 676c8bc638a0fb93f27d57c38c1ca00f6b2e234a Mon Sep 17 00:00:00 2001
From: st1020 
Date: Mon, 10 Jul 2023 11:44:01 +0800
Subject: [PATCH 078/161] fix: log filte

---
 dongtai_common/endpoint/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dongtai_common/endpoint/__init__.py b/dongtai_common/endpoint/__init__.py
index 0f7cfa42b..58ee0a610 100644
--- a/dongtai_common/endpoint/__init__.py
+++ b/dongtai_common/endpoint/__init__.py
@@ -124,7 +124,7 @@ def dispatch(self, request, *args, **kwargs):
                     raise ValueError("can not get request method")
                 operate_method = method
                 path, _path_regex, schema, filepath = VIEW_CLASS_TO_SCHEMA[self.__class__][method]
-                if 'dongtai' in filepath and 'dongtai_protocol' not in filepath:
+                if 'dongtai' in filepath and 'dongtai_protocol' in filepath:
                     return self.response
                 if schema is None:
                     raise ValueError("can not get schema")

From 7409e73204bd40e9ab5f8331aa24c0f48079f9a5 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Mon, 10 Jul 2023 14:14:19 +0800
Subject: [PATCH 079/161] feat: add new patch

---
 dongtai_conf/patch/__init__.py | 80 ++++++++++++++++++++++++++++++++++
 1 file changed, 80 insertions(+)
 create mode 100644 dongtai_conf/patch/__init__.py

diff --git a/dongtai_conf/patch/__init__.py b/dongtai_conf/patch/__init__.py
new file mode 100644
index 000000000..2149dd169
--- /dev/null
+++ b/dongtai_conf/patch/__init__.py
@@ -0,0 +1,80 @@
+import inspect
+import logging
+from dataclasses import dataclass
+from types import CodeType
+from typing import Callable
+
+logger = logging.getLogger("patch")
+
+
+@dataclass
+class PatchConfig:
+    type_check: bool
+
+
+PATCH_HANDLER: dict[CodeType, tuple[Callable, PatchConfig]] = {}
+
+
+def patch_point(*args) -> None:
+    current_frame = inspect.currentframe()
+    if current_frame is None:
+        logger.error("current frame is None, can not patch")
+        return
+    caller_frame = current_frame.f_back
+    if caller_frame is None:
+        logger.error("caller frame is None, can not patch")
+        return
+    caller_code = caller_frame.f_code
+    if caller_code in PATCH_HANDLER:
+        func, patch_config = PATCH_HANDLER[caller_code]
+        func_args, _, _, _, kwonlyargs, _, annotations = inspect.getfullargspec(func)
+
+        func_args += kwonlyargs
+
+        if args and len(args) != len(func_args):
+            # 如果显式传入参数，进行参数数量检查
+            logger.error(f"args number error, expect {len(func_args)}, get {len(args)}")
+            return
+
+        patch_func_args = {}
+        count = 0
+        for name in func_args:
+            if name in caller_frame.f_locals:
+                local_value = caller_frame.f_locals[name]
+                if args and args[count] is not local_value:
+                    # 如果显式传入参数，进行参数检查
+                    logger.error(
+                        f"args error, expect arg name {name}, get value {args[count]}"
+                    )
+                    return
+                if patch_config.type_check:
+                    # 如果启用类型检查，进行类型检查
+                    type_ = annotations.get(name, None)
+                    if type(type_) is type and not isinstance(local_value, type_):
+                        logger.error(
+                            "type check error, "
+                            f"name {name}, expect {type_}, get{type(local_value)}"
+                        )
+                patch_func_args[name] = local_value
+            else:
+                logger.error(f"can not call patch function, miss local var {name}")
+                return
+            count += 1
+        func(**patch_func_args)
+
+
+def patch(patch_func: Callable, type_check: bool = False):
+    def wrapper(func: Callable):
+        PATCH_HANDLER[patch_func.__code__] = (func, PatchConfig(type_check=type_check))
+        return func
+
+    return wrapper
+
+
+def check_patch():
+    for code, func in PATCH_HANDLER.items():
+        args, _, _, _, kwonlyargs, _, _ = inspect.getfullargspec(func)
+        if not set(args + kwonlyargs).issubset(set(code.co_varnames)):
+            logger.error(
+                f"error: expect args {args + kwonlyargs}, varnames {code.co_varnames}"
+            )

From ef60ea91042659884fdcccc8106cf514be9ec550 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Mon, 10 Jul 2023 14:48:02 +0800
Subject: [PATCH 080/161] feat: cython parallel compilation

---
 setup.py | 38 +++++++++++++++++++++++---------------
 1 file changed, 23 insertions(+), 15 deletions(-)

diff --git a/setup.py b/setup.py
index e6481e704..6589cf6f2 100644
--- a/setup.py
+++ b/setup.py
@@ -1,21 +1,29 @@
+import glob
+import multiprocessing
+import os
+import sys
 from distutils.core import setup
+from tempfile import gettempdir
+
 from Cython.Build import cythonize
-import glob
 
-from tempfile import gettempdir
-import os
+CPU_COUNT = multiprocessing.cpu_count()
+
+if "-j" not in sys.argv:
+    sys.argv.append("-j")
+    sys.argv.append(str(CPU_COUNT))
 
-dir_set = set(glob.glob('**/*.py', recursive=True))
+dir_set = set(glob.glob("**/*.py", recursive=True))
 
-dir_set.remove('manage.py')
-dir_set.remove('setup.py')
+dir_set.remove("manage.py")
+dir_set.remove("setup.py")
 
-setup(ext_modules=cythonize(
-    dir_set,
-    compiler_directives={
-        'language_level': 3,
-        'annotation_typing': False
-    },
-    exclude_failures=True,
-    cache=os.path.join(gettempdir(), 'cythoncache'),
-))
+setup(
+    ext_modules=cythonize(
+        dir_set,
+        compiler_directives={"language_level": 3, "annotation_typing": False},
+        exclude_failures=True,
+        cache=os.path.join(gettempdir(), "cythoncache"),
+        nthreads=CPU_COUNT,
+    )
+)

From cad87a2d66398d05d70a5d07ba32b7e30f51532a Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Mon, 10 Jul 2023 16:37:35 +0800
Subject: [PATCH 081/161] fix: max recursive error.

---
 dongtai_common/engine/vul_engine.py           |  74 +++++++++++++-----
 test/apiserver/test_agent_normal_vulnv2.py    |  25 ++++--
 .../mockdata/recursive_depth_1.pickle         | Bin 0 -> 4603512 bytes
 3 files changed, 76 insertions(+), 23 deletions(-)
 create mode 100644 test/integration/mockdata/recursive_depth_1.pickle

diff --git a/dongtai_common/engine/vul_engine.py b/dongtai_common/engine/vul_engine.py
index 570541b98..63c274e6e 100644
--- a/dongtai_common/engine/vul_engine.py
+++ b/dongtai_common/engine/vul_engine.py
@@ -129,24 +129,62 @@ def search(self, method_pool, vul_method_signature, vul_type=None):
         self.vul_type = vul_type
         self.prepare(method_pool, vul_method_signature)
         size = len(self.method_pool)
-        for index in range(size):
-            method = self.method_pool[index]
-            if self.hit_vul_method(method) is None:
-                continue
-
-            if 'sourceValues' in method:
-                self.taint_value = method['sourceValues']
-            # 找到sink点所在索引后，开始向后递归
-            current_link = list()
-            vul_method_detail = self.copy_method(method_detail=method,
-                                                 sink=True)
-            current_link.append(vul_method_detail)
-            self.pool_value = set(method.get('sourceHash'))
-            self.vul_source_signature = None
-            logger.info(f'==> current taint hash: {self.pool_value}')
-            if self.loop(index, size, current_link,
-                         set(method.get('sourceHash'))):
-                break
+        from collections import defaultdict
+        from itertools import product
+        from functools import reduce
+        import networkit as nk
+        g = nk.Graph(weighted=True,directed=True)
+        source_hash_dict = defaultdict(set)
+        target_hash_dict = defaultdict(set)
+        invokeid_dict = {}
+        for pool in self.method_pool:
+            for s_hash in pool['sourceHash']:
+                source_hash_dict[s_hash].add(pool['invokeId'])
+            for t_hash in pool['targetHash']:
+                target_hash_dict[t_hash].add(pool['invokeId'])
+            invokeid_dict[pool['invokeId']] = pool
+        vul_methods = list(map(lambda x:x['invokeId'], filter(self.hit_vul_method, self.method_pool)))
+        source_methods = list(
+            map(
+                lambda x: x['invokeId'],
+                filter(
+                    lambda x: x.get('source', False) and x.get('signature') !=
+                    'org.springframework.web.util.pattern.PathPattern.getPatternString()',
+                    self.method_pool)))
+        for pool in self.method_pool:
+            hashs = list(pool['sourceHash'])
+            vecs = list(
+                product(
+                    [pool['invokeId']],
+                    reduce(lambda x, y: x | y,
+                           [source_hash_dict[i] for i in pool['targetHash']],
+                           set()),
+                ))
+            for source, target in vecs:
+                print(source, target)
+                g.addEdge(source,
+                          target,
+                          abs(source - target)**1.1,
+                          addMissing=True)
+        for s, t in product(source_methods, vul_methods):
+            dij_obj = nk.distance.BidirectionalDijkstra(g, s, t).run()
+            if dij_obj.getDistance() != 1.7976931348623157e+308: # INF here!
+                logger.info('find sink here!')
+                path = dij_obj.getPath()
+                total_path = [s, *path, t]
+                final_stack = []
+                for path_key in total_path:
+                    sub_method = invokeid_dict[path_key]
+                    if sub_method.get('source'):
+                        final_stack.append(self.copy_method(sub_method, source=True))
+                    if sub_method['invokeId'] == t:
+                        self.vul_source_signature = f"{sub_method.get('className')}.{sub_method.get('methodName')}"
+                        self.taint_value = sub_method['targetValues']
+                        final_stack.append(self.copy_method(sub_method, sink=True))
+                    else:
+                        final_stack.append(self.copy_method(sub_method, propagator=True))
+                self.vul_stack = [final_stack]
+        current_link = list()
         if self.vul_source_signature and 'sourceType' in self.vul_stack[-1][
                 -1].keys():
             final_stack = self.vul_stack[-1][-1]
diff --git a/test/apiserver/test_agent_normal_vulnv2.py b/test/apiserver/test_agent_normal_vulnv2.py
index 5e6a2a422..c66910e4b 100644
--- a/test/apiserver/test_agent_normal_vulnv2.py
+++ b/test/apiserver/test_agent_normal_vulnv2.py
@@ -15,6 +15,7 @@
 from unittest.mock import patch, MagicMock
 import uuid
 from dongtai_engine.tasks import search_vul_from_method_pool
+import pickle
 
 with open('./test/integration/mockdata/new-exec.json') as fp:
     new_exec_json = json.load(fp)
@@ -31,6 +32,9 @@
 with open('./test/integration/mockdata/xss.json') as fp:
     xss_json = json.load(fp)
 
+with open('./test/integration/mockdata/recursive_depth_1.pickle', 'rb') as fp:
+    recursive_pickle = pickle.load(fp)
+
 
 def mock_uuid():
     return uuid.UUID(int=0)
@@ -46,11 +50,13 @@ def test_agent_vuln_upload2(self):
             res = self.agent_report(new_exec_json)
             search_vul_from_method_pool(str(uuid.UUID(int=0).hex),
                                         self.agent_id)
-            print(
-                IastVulnerabilityModel.objects.filter(agent_id=self.agent_id,
-                                                      level_id=1).count())
-            assert IastVulnerabilityModel.objects.filter(
-                agent_id=self.agent_id, level_id=1).count() == 1
+        print(
+            IastVulnerabilityModel.objects.filter(agent_id=self.agent_id,
+                                                  level_id=1).count())
+        assert IastVulnerabilityModel.objects.filter(agent_id=self.agent_id,
+                                                     level_id=1).count() == 1
+        vul = IastVulnerabilityModel.objects.filter(agent_id=self.agent_id,
+                                                    level_id=1).first()
 
     def test_agent_vuln_upload3(self):
         with patch('uuid.uuid4', mock_uuid):
@@ -83,3 +89,12 @@ def test_agent_vuln_upload6(self):
                                         self.agent_id)
             assert IastVulnerabilityModel.objects.filter(
                 agent_id=self.agent_id, level_id=2).count() == 0
+
+    def test_agent_vuln_upload7(self):
+        recursive_pickle.agent_id = self.agent_id
+        recursive_pickle.save()
+        search_vul_from_method_pool(recursive_pickle.pool_sign, self.agent_id)
+        assert IastVulnerabilityModel.objects.filter(
+            agent_id=self.agent_id,
+            level_id=1,
+        ).count() == 1
diff --git a/test/integration/mockdata/recursive_depth_1.pickle b/test/integration/mockdata/recursive_depth_1.pickle
new file mode 100644
index 0000000000000000000000000000000000000000..00b8fab84f7ebe8226b53cb3cc8f37f036a77f2e
GIT binary patch
literal 4603512
zcmeFaTaP5inWm|o?VTBLW)|3sxi1tNn3AAK+4)dFW1A<5IdZ)rRHK2}i(Oy=-c9W<*uSzrV6XSr?CYMM;pyq-Jji8Z?qe@!)MQmehIx2I`ulv3
zo7wkY{_g+!fBvT*wEuklA5T8po67d8m#4>{
zpY2}#SNzq<#rd=S_VknEix)30&R>1~+Pk~`^NW+;TwI*J`afR%@2`IM>R(fB)&}?(F31ljH62^Bw-?>wmpHIXOLl_Uixq
z`d^(qef9OfIoW->y*}H&x_JbU%^e|o;(
zzkK|YpPXGBZ_l1zT(~GGo%i4W%}?Z!#5uU!
z{nIDUciWTQ<*PsbzyHGze(<4+*C#*w!}ZCV^6H}x?D@&hp1F75|NNK7KZ}1k`I&tF
z^i+zcYJc|hvzMPgdH=Wi-4}0v_3nE@e)z#}?XTYd-HQ*^+xrjS|4=`9Q-1d3FF!r~
z_4~h-@4ofMizn*mzx?Q(-|kP|`Ni`O)gNAd^w!0n-~HhCUp;yMZTm#MYo4eN#gm`?
zR{Z+?WBJkhzs27ln9&PdGX}4XZF``eJ)BsE`s^1ke*OO2Uw-)6
z@8yTjUS2%aPsFcZeEG$P>Ze!R<~+Q0@fSS)=XYO!`PoxtuRr?WiP(M+uYdhwfA-NI
zU*CWD!Iv*jUi|)-AAa!UvycAxhWOP7f4IhZ{d|k_`Rf-aZu`gAuYUfs-<^H*;>+hJ
zKl?rY`xjrm`T3JCo}T~i?CJS$_kVcj?9U&)|ML0qJ8xWVKX`)YoPF{1H2%{uKAShs
z|L~WO{`hl@and~3{r>iIIs1jg&%b>7!SBC#@|F3^yQk(aZ$A5nf9O8qm)pzjiz|Ft
zk6)bp!&&nkXg>H8eBWMu{e$*7SGQk-umA7cug3!=#B1+gZqKhi-Cb_pK0m%_zUYsS
zpI@JUzB_sCm%H7|&Fg2UU+f<5PhadVuJ`}o#A|O}oS*NG_oo-$!Ay1Z!g}!)hACcPQH5e{O`a15B=BplTWW+{qeuxo80_DJQ)4n
z{I54B;KA*CeC_z+ydCCjhpWT$$1=Y562Ffxw&$Du#pk>8$0yc!wN-LsweY20>Zw}&t{M0>_#-B(Z#fehN
z?M&S6+|!`cNzk9FPotDit$q4hdo8(lRcYR0;v=~kagKBT$;;-_`^W#!|MUkR|MSDEi|fnd-NVNZKHZ*O;Q$_&>66-V#wFYzsIe(vh@
z+4**VeVKg1(be_StM;jna7q4Q^Zc8mzJ0~RN$?>V|+|LDrfNhaK_j_
z;b-^T%V)d&H#`ABaJIXAvpMGG?Oj|xJGy$=9O;383(YrXq&9B7o
z@rSEs=$lvm9o|~=K5!zR-yFrggTOQKoBV9oygHtA-kleGWS7@h``tGUt{kF^hicPx1SG
z^68&`WWV26vi(eNU#K}B&AA_aFYiC==zRC({_Qh_{Q3S-;=6h0;r0cG-%hEmlfehw
zpX%@a#ca=ddpvJm;2Yas>DTi;JUhMGH)rH;^&R{^Uy^3r#P`#xXog=<{6>zif8ray
zpZIbjX3w`r-If00^5}QFf4bgX?SHyG2HX9kGur6gvHp`)%~1?4nBV6ujK1fiYwVXt
zh=niCcKf6D7v&xNw!C-q7u^xH-^cr3@NfO8qvqn-V(QhS`@i>z?|=X7cDTXUKk=m>
z{6c((3-OW28=!lmcT=?JIo}XXrA6#CaKS0g&4DtKmKR-M>6H;m9xeut+
zq?Af(X{41-dKqK{&q{!qR$x8llvhDTEwt21Yy9q6?X=fHMDx6{^eTuyK?)C>#TZ#0Ypn{5+oCzNO|(;a%Zv;eFwa;ho{F$tV@xJ08S4
z)KVD5OzZ+c+y;x*%PYUNi^jmGoMABfXU#9|lfQ
z0Q$%vnqv=21}%e;!OGBto(wo&I6VSy3nxiNEu)bU#O?s?GUB5=Iyy3<`4OXn!HLC3
zWP@?Rc${Q>T)~Rw)vO8@Jdg8^&&vht@d4n23r;ixITf4^&IE^#1s|Trhl7tWc+ngK
zz8?5C;KRpfjgPkZBt!FI;?u<8;q$}?+6;zQ#E&%}Dn8Z_1&%yK6`~H&pn3=qRRq3}
z&1Vi>(7Y+3qR>&8C~Ooi3KR_Zx#sh4-f#PH;WNRB$JxaxY+g@Be0!Typ;S~lDif6j
zy+#Fw#b^=Vwa>@F`aF8=|){BLYe
z9(;OwwnOQ2b$ogH5;*s#A2si$x!w$tk{1SE7R~r(j0s9ig@X7q}LgC;m(;!6mdYR?_wEr
zz!4HQ8-6Bi-oj?k_#FnOg5psjn4oRYE@=1}U?6^P89DzwM^3|F&r&iD4|Q~#$Q$@WIfH^S<-ijU@Y-Wy--cULLnar@X`iKGy`#%k?+gR9?$q(6W0
z@~jzWN_Iexcdyo*lkciH@Z4L&@m&PR;OS|-vA=lpMiJc{EM}zqx0^s=Z{EW<&!5l^
z?_yv!m?g{-W(l+8UY78(z{>(J3%o3R%Pq8TN65qc+h%_5kH7aE-S46|)6YpUWji7A
zhS1s>+nDOw-u|27%JD0g<;!NmXnWuV_)=R1UwUU2@a;S;F!b@~?R@2j{f)|QlKIi@
z%iZy#+dni9x6=Tzw&&<{i|Kgy-&0I&fBs^NiPOz`nyaJNulB#$U4Hr|{(#9)N89(C
zwKmP4+NCz1$Vc~{(7fp5sN2`Qx<1EDzfYU_u18NWGyjX-&D`c@7TVR-#br}R9NkP<
z7mse9@Zjbd-}lG6pH94ddpu5!Uhu8m+3p#R>HJyZhi;DP(KkM)ePMI+Mw|b>xs`qU
zc5hy^dvlMvKi&LD4DG=*^Zc&v%~fxvNWN&-CSfA|%a<3I`=g&@KL7Fd>Dlg`?fD7j
zx_2*ia?w2S`R>ugwuoOUyz?;g6-w`_UyYHi}>bmV&b!U{pnS+WCu$!?tB?<&&bUIJo?7x
z+?<5(_MyCWdWAJGSW0l&ebPrK7q@@+==MMQ2X8)b;lJA#aQyQp7tJuI$GfT5#(cZ)
z^x6INcJ0aSVPisUbHTg){TV&p9l3CS-OHa`G)svdrC%1)7~5BE#+aTk-p)U7-uj)X
z=SS^fVbRg);gG-SMBko^2@ttB|Mc|P(Ys&WKMRMyY0rY`
zzo_dz3sYq9Y5P6CU8K^kGyHOY@0{Gf{12bio+nIdKYWrY0O^ML_4)A*&W`
zNe{mZQ%`J97p4jD*WtTdobLqHFlyi1g9f*kCm&pHF@LIAi1fwow(?0ncJnoj-x993
zKiXZwk}o@W?B;iC`c<1X;#m6Gtcb^2zt`{n=?xULw}(7_pnYL1v^jiFlGnA{<9!D(
zmOR-0ftwSc`%>Wk3u49A7bw2pZkAj8Zr2oogI65={momysdB&db9(z6i-LE97rZ_8
zyAQRm>H8Al{xKfkERe^keDmU~i~a7l47dfq;or8erd=OBw=dWJY`g9e8yGbBDP=dU
z8n1l&TKE6{=GEdDNp4>4wgfr6(63IPJ>MU79Db-!?mgVTs2hJMqt|SA1o?Y>_qq)N
zx@su(`t4J1NE$1O?tg*qE|ES5?>#xac>Db1@84??@NkG=KX{u*cb;+cs#v%+
zIt%w-_TEdiUv@0idzAY7_9U2Ze98UuODrnAI(onTce4}B+t5na%`X*R^#d%r-(5aR
zJlMW;v!3dkPQ&zz|NYG?H)Bk`I@Uscrx)K|eSUL*f8SoTP~GF?o;RB1=H(BMcSX?6
zAvbT}M)TbK@SQ`sc^>ww_+h)!^F~#4yD(^b|1NL;LKwE${oxzSBNHbk
z3Utr@2F=?&<3{xL!`w*L_SD??rQ7WQ8X?%e++dN!ShS93u_h)8OyGOMBew6c-Mit&
zXEke8M{2Qo(l;(lXl9W@f`0R)(L3ji(?z*>HMw||(?z-7b+q2~Tc(RfoxcAvy!4y<
zmww*K=be1s$>*JXPVlL9rwqadVPgt5rc1w@3hvI@x14L_v2}>^*tbWH%f57JcyD&k
zL?PI2A=_>UcZIt-Z?=DYAKO6v9=r106no8ff4^%cjH4jzH{~0TiStx)UU{=?~~MlvXcF391$EINF2iU{yUY3%o3_eyw*S;{LKQF-PXVr6|RY8Q75=6RO9R>O?u4
zw`5HWx}Ms#uqhq2GGWp%X|St2!=}=(X_MxgW@G+!Ni*M+ZqPlU>7Idx0@(5v|Bc2L
z;{Y0DQ{LD)47HxHp(v#&Eh0*pDE0PH=@Mn0C>^ylBd3y6c~zR=X
zmjzxHCKnU_b>-`AXQk90#J7(%vx{yvzCCov71)EnY4Dfmo;y{}=B-&%qpqiXE$d3v
zsQV|whGFBI4xH4bRo=31E8DjB=il_M+8M2tZ`AWfy=B`ycQOvZ0JM+9TBJrifL16+O&18S)eG@T4*ngGghGOxWith
ziOpYz*_1bS`mxrNHdLilrA1UJBSplvn5}TGM;9scR4E%e5mX7PQP?P4nU9ei&FM;B7ctfQwPPYsWfX^
z{*((8rA8TxFDCqTjL~BUU8%-1O7A99-q5)nX+23pNlHmtM3OR2Vw82q$vjEQW=@1u
zLMryUqbJ3cG)C1!DkUi;=|GYaHmZk>Wk}K(l(tc8H?C4AO4+;xYhuv#)UIVMsbnW7
z0_X@p6@I}fO6#m_8vu1-ftGZGn>vS~){`~#r1YdkJShVu
zL{Vw=(gn&aPb%5RiI_@EjY3nEqKzl6q&=q6lhTt8JSkzLde~TqCzVzTt!1K*(?l_w
zw_;5Ux}Ms#uqS07Cj#gQKxM$@k5r{qR<
z${Ra}q1F>N6r~iUMMNnR1q-k$Enm7snI}rw$cdavPK^Tnlj%yaoJjPXN>NHtIuNCV
zjp|`z8KP7MA+0vTj3<vS~))O@pr4*$_L@D#6-c1F&Jeem-*~y8NN=n6UcXXvO
zgcxE^sT8FYr2|n)*r*;hmLf`xLHA)R6P+~D#cbY+bv)*JO4q`clzp7YpZEDwLP0TW
zD$SaPKW$u~CDqYjM*tBGnivFYUC@d2k>PGK7PR0*n42)fc3K!BkKRf9#iQ_=}8Bkl(11fY%If*N)x3)!739S?ld7OibZF{iR7|aC7=}~c
z-08B}!bbJ5
zu@qOTqVmSs#H7-xf;Ml_ni_RIVu7|)3+*)KRRnKs
z#Lc=W)p%J0ZsXlar@Xn-kF}n(p)92=Eh0-9DfRYZ=^|yGEM-e4f+|5ZL{DEDgMlyY
zL6x$UvUDIz2^-bJ#zJJN7Z9a7wN>0iIh(g+O$@r8+O@DLWkV+d=mX3H(n`^>CR|wZ?#v_Tj3I_^lmogjU9PA)Ox~(qLiYvh$v;E)Z2@tOO$z{lr5de
zspQlU9bIV*8m_eGREkoH(t#)?Y*Y^$3lXJQ0;#l4%;=daX7g68sZrNcycRa4Z0bY;
z9TBJrl4(6}X3s2gNc-q<+|
zwVtq{D5WSZB1)Mk^|oT^5@nt!Wk)A+DmgVoLsuHx779J5Qj}7Z4n!$oqk7m_h$wCD
z>0raS#B`pGO;@vdOV-q=>#1H#s#1?V|8S-*o__X+?b-G2;o}DnKR!DXQfXrycEgcE
zN~xrl2J2m&^b#SZ(1BVhrIk@uIfWi8Dq`;gsWHb`8*Q?~simdz@wt+Cc(_Xdltxt(y*DW@Iwy>-rc7hLqhORv!2nuTqV!d3;zxT!R58U)p0
ziK^7aW>WxzD-LFqs)#E`B7(4qX`fDcbLTMBdcuaPl!?MbK|2r;oD5C{r-L)W+2Ft(
zKDYq-2wnuB4z4o*8E^|Q1ytgFz{VdQ9aRMO{|G?^Tqc7F!QyA|KxqDnAqrqqz?CXu
z8)bS80k6>|$_!NsPGv(Us?w1vC8yTg->K(Rs#2=bfhy%?ftQ5`50B5bS65HAFIuhj
z+4hU=(b@L=+0lFZ%hU5`Z(N_Ao$M}~GWY8A+4**Vefdq#JKA5|{M@4-Hb3%Ww|{V~R5uglY~GqRHR^iG*TSxp4V}oKBL;;N
zXWmqrH!XwO1-eoMkHW1M7jZYHxr-95882%Xrc>VB>Bm}6+EA8KmKKqvjFfsiv2>9#
zPnNQw6G4@r%DXU4N2})EiReLvJ4@;LP^E4^jIgwIHshqDKy}vP~J*85UQj-oeDPf~}*jR=pmC8rV
zrm=QBp(kC-<}FyqW3H!iEi6gd%Zd1TA3vM*a*Ucvqo%=66&EN;H!4_*UQQazUg@HIhC4}
zn)I$FC2SBjmZ3?p`6TvVQZX@|XR4abTe7A`T~GB|Se3G&6A^TTpxB9xwzSe7qb-8E
zC8|=RoJ5x`7o^ix_-GOdGzh1>v2z$|Jy}CjN>f_Ilrm82-LSL^lzFC<{hWxY#8hsB
z9KF4fr9Gz7l+u(AOetZbde~ToDOKL0nU8aNJfmlmE@tyqtcgL_Q@fV+q=DU>2%sYX
z^-aeLs?sX&%(Vcj7wAd76~SOSDdv=FiDqxYCHmeEqA73c9EMs?*3gsElNRx$43v6%
zJ9U9F%aaB+b0Ve^Q`s>tdUK;ndrYM#r6(PDQo=^{u(1qJDz$Y6P0mF^lulH$c}v#B
zpzA4K3yV^=aw36_2o$Z_m^PKBO-rC+fua;!l4?vTMIY)Yl~X>MKf`Rw8$112>q#4`
zQmWD-s+5tULaaMd=BZM)aw4b_R73Ezr8l;;2UV(4s?vcfC2Uj=8_Q6oIyxteQ*u0|
zbfTQiTe2nwT~F;=*p#xB69IGtpf+HEGi7O&m2Cr{(l5}IZi1FpV&T0A+M(%ba4NA$
z_8^<`#?E1=^@I&YDMe`!QOZP-)%L6I5@nt!WiuyoDmgU-M^_rXhAZtkm7f>9>M#JxH+dg{|DZ|)q1T2I(el~R=!QKd{2QEkuaE>Y&GQZ{rVr;<}cu(YMo
zE7;PWQ>jX+N(ZWxuu(m1EJT$?wAGM7nS?5xsAlt)teeEB>#1G~t5P;}B7%+(R0qMh
zsWfg{1eFU^rQYH?ZzVe3i{Q0xgy=Yd^xbgE8#{-g){`|fr8K2QOeq5;R@=6^3zT`L
zlntGTsl?O}3}tEb60)?%RGLzn(t#-@Y*Y^$%P^$|7b=fJHID1liDEWy#hMs&J+*6D
zPwLswi2ynRP!$BjrqZx!091qpdeV&)*dW{EpWuv_O+%+d01dJ!Z|od~T2I(elv0!y
z5v5ENY%f@MqRbMdo*kXYspQlUG+k-*0W^HeE2I+0V!sUaxZ(&!>=Y0s%trBtN@RZ7^X9yXSt
zN-fqANsU5vJg0OsQO)KpSyQ8~r+O`{O4-wi2s%Pg_$bCrrE$|Ds9K;ZH5yYYbim?!
zYoqmAhQvOlgK)|lJBOjxlQlG@G^IsMDFdb6UMyXp%rm8I=|oH=riLIXOR-BmWNDA7
zG^I4915--as2(^X-
zs)r4fWXEUQtE*d=dB5s>TLjgM6s72j;xuZ7=Ko-FX|ku_-Ehhq
zJN|a4^<)iADNSh+Q_4W8w--wnD6>qdV@oGuDls)+M<~itG?R9&$5fh9n$m$OC2Uj=
z8_O`IGAOUKb0VoqlhthAl68YG#=R1HJ;iHbQOb@^B+wCoVsQ)8rqZ-&2~;dllxk(I
z#8grlw1Fs9PMYzpx?wiujh%k1^`s3|DOG6^Rmw=Iw--wnDf3h*TRIU`399I2Nn0A7
ziO%+*N>xf#I#8vAjp|`zA*$3XY4E?)K5-NEY~GqRG3a`V*TSll9i2#^BLcP92Z*+`
z&NibBfd;=sRf@~JGSX^~jY@@&R>XuV9b{A9*pY{^){{0=rBtOwR4F5+-exRaq|8&L
z?CC^MC8!4PXiJ-EQO@+BN>xf#I#8vAjp|`z8LAY!N?R?^RAXGJ>U24qw`5HWx}Ms#
zuqkCrCj#gQKn*+;WoeylMq2!%9}ceq1KZ%^rZBp
zMLa13rQSv?U7*bKr0nNJOeLlUZ>dV7Q&6QnrqYwrlMXy7VWWE3ScWIXszd3q5K<>P
zI!#owc}v#BpzA4K3yV_rb0UF`2vqyXw5c?0S^|{|6s4OOqcT=v{h86&9m4AL>iav{
zls9(z99&P@P?b`Z7Ez^)lzMxybdfSom9nK1L6x8yusJ$ysgtm!J*ZNZQk4!=DPf~}
z*jR=t_1LFWY8MlWA1BJ$yd`U5(Dl@=WlgDNLni|02tbvO44X>BrU6hN7HCS9bONq4
zsDPCvLExWMlbzC$((oL`)^72Cu0~BQVg(
z9#iQ_=}8Bkl(11fY%If*N)?5$PWr@@o{4HUZ^@b%bUnpuVNuG4P9)F~fl42lHkGDL
zOQ3dvqSQi=N^Ddr@NZ|E?`L93!!Voj#!f%hdeVlfl&Z9dDrKb9+li%%lzFO@4V?(8
z1l8abZE3U-wzLOTs#2=bfhr|zR1X^qQKim#Z9_C}yiMsuJ)5^?O$@r8;Zg`e=}PHJ2dqJZ?rUn-&OQUs=r9Gz7
zl+u(AOetZbde~ToDUCj$UiBfdTiir3o3~<347#4$wX7#KZ0ke-9RVo%`Y>!N4VxA~
z^#VPq7208&(xwBfc3Rkgz5Q{S-)}FU?e_0%ubw}A{NUq3H04d5!%*wV8hTQC(juOe
zfl_ZjmM&0cc~Zl+PQ+AVYH)$7G-B%|>v~M3C#5GHcv8Yf^{}xFPb$$^1N}8Hy=Ppg
zPE@mbOV-4o>nUCfi&D0AB7u$w)M42
z45z%g(~q^DxS=bhD=p$mnJM))Wa%_i3~F{lNRQJB`*ZM0!f>z8Otof9D{
z>qS&j1}mjV+^&;@c*+|*{aEXX8`@IZ(jvB$nNn{{mM&A~*;00PBB_#8gTvYy3R4}e
z=}DEgl(uwWO9>m*!^SdfsZqj49aLfk$Qg1(&E>Y&GQnqy>
zr;<~#2N7*)v?^NNb1GFSRp~&L5;m%bjb*4(DWr5J3aQ71>Sm&x&0DjkMqN+&TG*Ac
zuM-(`#Gu*-=1rw}(=w=BqARt^X(N<2y4|$POB2TtG>oUb!PAemp17ecr7bODOPMM4
z_GIZYWu7f%YbTN_Ni{e_VH&N3Fzrc|wv@JXU`q)b)x*XzY-z;3EsOpdiAr@dQP1YB
zSyQ8~r+O_bOEsH25kW@?ipFh>n@Z!RK~NJGC`+aB5ex5|4yYEl>m#1D}n^HD*B7lwnRQSNKsWfa_096Y#
zrJIONvW>-j0~JspQn)
zBwcAV0j{*?REkoH(t#)?Y*Y^$%MhhfTWf{K#>zOJ{r2+NZvW2q>iNUR4?do%X7iS;
zsZrNcy%tubZ0ke>y^o+i`oOrUG;SIM)p3EURGXj$vd7tGa;X*4j{|6sO?gx2Fw}aY
zhN6_Bw1g;So>bh8rOT6fqLfXYNU5aM-~?T1#3oE8^pr|bN>O@OloB=w8w(Mo#%PD;
z!#**kG+oT*tysrnuBUV@Y)RSBiTpX@&)}IgwPsDrpKghkR7$I~*4|<{gVF)pLnJim
zAer)}jy#ODp0uGTr6?^TN*O8j?k(6w$~;lZeoh2cf@-jlt~54%qkRvm6r~iU15rxY
zs2(;JB1+ME*a_i%qEyxCdNyy(nizCF#cNqrs@Tqn1Ue#6?>*C|(zIy_R4-7KZt$(N
zp=sr0gHRSVgR%)%ItZt{xr23ZT@bD(ZsRE>))r+Pp<;YSi_VuZ3+XdpeOpM-1wa
zHWa3H_8Dy%R4mYzLf}L5SgVwA&BOxCG94$79Hdj;-08S1FkvJ?}0V$d>Cs!r6id280hpzA4K
z3#(E#bs~X|2-JGdw5c?08Uj^*fvR-lyfw}^q0rDtDi?g}Zk=pqogTs|Z|?MCttW2i
zO6f|AxKd_Hz5Q6aOqu6O+181qN>UBhQI|%ep-X#Gr7NW?9k^1$M)k0<5LX(c5Y7sb
zScE@S(B>^#Q=_h@d@XEC+1H5-I$}_R15POJCQ+04640n-c*`5ErZGh+EO1(
z2=J38C}D$)J|^zh$w4~h&7FR%^`s4DDP?I9S;|PMw=qi>Df470`#TX-397*w`qF45
zd}$A=l%TSx>1
zD%snKm`Y3y*p8j5)Cj239#iQ_=}8Bkl(11fY%If*qFlA!z@*4=p*m5`<}FzhgRZA|
zEi6je*@*-?B2ekEV-lPX;)UFpD;5;m%bjfJ>UOe&37
z3aM&#~M+_=(CxpVZ&W58cgQ_LkQnYk3*ss(^xYDSD
zk=WlK*ZKYS^4V_x&i3m0!^aOkmVxmoMQrglYwv?GtZ&#KsQ|8%HHg_Va
zl2n706sDj*2-BWaX-jEK2ey>3Q9W!d!Ph_+p%k1fan)Y-$ag7C_wsE$K$9fZef`#=nDC)@mEZGfD^9ls9$`
zL#-!lC`u_xi-=MtO1-UEx@AW!bbJ5u?$ff
z17@3PFWk6JovLQ@maM5!*HgWgRi%PGors_#1ohrAZYqtN7D4p_RcVM0x5R|j%}&{v
zT7m7V#tAeCr@XOq7-~IPLsLpqTEvtxQ0i^Q(gn&aQ!3cgiI_@E4cPRKvQ%5h(jHT3
zN@+?5rj)QzJ!~w)l*%ZvrLqpGwf7U%Y~GSJG3a`V*TSNdJ)KCPBLa2aF>NYMo0dSu
z0!69UMrbFL^cK67V)1>dr{FM~^2Sa-)_T&0s+6j~I)Z2@ti4M<<;~P3f5^XY-byB0R3?B_%P9RaBI
z*b0)e)Yd3V0if7=VTq>H3T3r#w(@s~A8C~wPbrmmqbYCd9EMs?*3gsElNRx$43v7i
zuylbk&y%v56ET&T8jPbVjT+rNO^>Pcr1Yc%PfFOR9yXTYNj0_!@#yBH#&xPs7qfXQ
z*2JLesa*?uQucBpfQ|svc*n4*G;CS`H4F5l7I)@^)KbOho5fdF=yCoGqA73c9EMs?
z*3gsElNRx$43v7?uylbk&y%v76ET&T8jPhXjaosK_LxdfN>4iQq=b#?VPhGdR7#l~}(UdoJ4nwUcYv@VoNsD+=21>nMSh_%&=SkVjiI_@E4aQKF
zMlGRAdrYM#r6(PDQo=^{u(1qJs=XIj0cjI;>O?V{w_;5Ux}Ms#s3#4Pt(*v;BLG$2
zF>ERgn+8BbSfD4}D2GK1R@r8YQg~7s{rEoJAeu6#&TR^=Cu`_Q=}C)tQU*%BJy^Ox
znd3=AWGg3PDls*nF9=ns7Eq-apf>0&l-#hR#~
z*HgO|_N46OL;xKDsPv9uQ)$?=0BRTLNwp5nD1(hjBbo?GBW0rH{UDn1rp{rg^<)h_
zDLrWsPs%{4w*gBRDDyli`#2F(iK)RTs?vy!m$c|Hm7bKIbl^z|8`Z@HHv2z$|Jz+ypN>N%wlrmB3?ZDC{$~;lZMo#2Za%wP=t~9CuSK4zbMJYw;K$H?T
zs)vnbh*B9{Go{BRCiF~Ivw2I_)TryJUJI*IwsInZ-bYXyy<^-|8aEAs`nW(N%wlrmB3ZNkze$~;lZ
zUQXmxa%#|#t`ryuSK4zbMJYw;K$H?Ts)vnbh*F78FVYw<6V05a%Gta%YiiW>l&@u7
zX<$1iGU$jwy|>JpO7o^=P`yA`>S9zDepF(gQtzcwKA}p7>6ABj`mxrNHk755rA1^Z
zBcEM26`lBIzSod~J~)u01?DL4?mvyB7AOZ0JM)9RaBI*b$Pd)YPa-8vu2Ffu7W%-D$H&
zsSUzgDWn&PT}lVhls9z_L#-!k=t=2Gi+EB7O1+I(x6Aq~C(P~{i_(c|HgCzA7<4_wYhh8!c1|SF5rIM{Fl{PL
zo0dS$0!8Tt>fTC)y~+iaUPa6+jmfq5cf%=f?({*pp17ear7JDsN|`D3wqofrWu7Z#
zM<%=U2xV!?S-|NSBgdU(T?xd$w4~h&7FR%^`s4DDP?I9
zS;|PMw;f9tDf470yE+k6393OG`qHQ*d}$A=l%xl-_BlAx*7v+Qe}H$w4^f
z&7FR%^~4QbDP3t1SISJOw;M~BDYIOuXHzGVDoHhHO<)~ty^*HgR}R;6t1L;@WVXwy4~wzSSI^jiW|
z3sj}SOU(7fyi%vUw^0b2SlFP%-E7JmJN;PeNgJwCs?s8=l#xq#4`QmWD-s+5sZZ%dXgQs${rc6K7D5>$i2y<*x@2V2^MDpe^}=|GhdHmZk>WvJ4K
zV%1n&u*ZEhCd%2oC2L~P_0+CqO{rsJCj#gQK)p8%n@Yo`1yH>}Q@TMz1Bu1=!g_=D
z^Z17P#9pO?Y|0xuhoROJHWZ~4rA0(36Q$mMEM20^5~Yr9oye)=)S$4KhOX2`>3UA3
zD5WSJh*H8v^{}xFQ7VlxnA78;9-GpsayD
zVu7wy25XHE()nl_Q|eseUfnRA^5#xI)_T&0vXrv4h%9BK)Z34xi{|ssz=b
zJbh`DCQ93bDrG5U=|GkeHmZk>g~(E^yfID&F_xth^=#&vbzDTRr+6)_O4-(l1Ue#6
zOx$DIRGKyofoi`%RqC7<-U)|A_(n%(luD`6K{jQMox@n`NgJwCs?s8=l#x
zycSlaZ0ke<9TBMUhG|o2+Oz~}7N|-CzLQ#EMyaw<6CvZY|0xu{aEWs8>&*O
z(juyqky7stgI%P|Q>AR{L{KHD24!hWqf`z2HTU#eP^BuRDjld&!bbJ5u?$t(T&Qf2
z()w|!I#JK&tyvR;uBUh{tV-F|i3B<#P>osRw54^n8f^(wE>M+jOmtc~YgMxZ4NLGb
zzjPcx!*I%*JN;Pei5t37y3!)9l$lcRCWBq3%yXse>qJr|sRm`JOC#oLN!61oT`67Z
zz?Bj^#Q=_h@d@XBBEn7R0K}QUVJvNv(mF7*upeiiV
zmO2GUHXhrA1|zNWsZC&p@su}s`mxp%H?*a+rA2HhGo{|HEM2C|vZa>Iok*%A)u1$m
zX_Q!2Bzsb&Eu}3T*iynq^{}xFTZ--Cv=1n#$MbqNCSA|wtyxo}uBUn}EKAwji3mDE
zQ0WchrqZ}+5!5bFmZC<`ScC7e49zHQq!DVIK!b3~8#{-g){`|fr8K2QOeq7U-j*y~
zpv*I+?CeBLC8h>!(oR__B4lZgsWhcDr2|t+*r*;hmSIYx^4dx)d>_}T6UA)aiZwCl
zdTQ6go|KK92%sYX72YsxDh-1mJZS>Z|)q1T2I(el~R=!QKd|jdi$|-i84=>vaJ(2m7E%6q%DoY!It)%
zN>xf#I#8vAjp|`z8LHGdC((#c>x3$usAlt)teeEB>#1JLs#3$QPDIcVf_kqRHC8x6ED7sZ^y@r2|z;*r*;hmZ3^@5Gr^~EKR7=iE1`)$+}65x}NH_uqtIw
zCnD$wL7mr(n@Z!RMNqLoRVtO0+6%9&!}5C@j4|T`8iZ5c*f|Wfo~)rMr7109N*O5i
zHe=}mWu7TzPbXq3F*Qg}SsI0bEbTFsrj(|1U`h!a)x*XzOeuCtkc@#{HgCzA
z7<4_wYhh8!rcNZ#5rJB-nKqTCO+%oQBSJ9I3g?4!7TcC4MCmY_^2Sa-)_T&0
zs+6j~I)Z34xi{|ssz;_9c^h88n(0tRjN{|(t#=^Y*Y^$%TT4*Ed|@9
z=%^B<>O?)8w`NTYx}M^-uqtI+Clct0K#kW-n@ZEBB~Y_KRl4yuU^=NZ)=96Tl2AV5
zNu|SZ%9}g=SnG)!x>CB*BCeE~Qg1(&E>q^YQnqy>sghKKwA7_hDCp9jROw3TN(Zi#
zuu(m1EX0*M7orkI+Ht8mRnX=wT2rI0r+h7JOWD_n3_4;^?KShJ(!6OIR4&n$I`5@&
zMoDY6i&_hpxLqd)@su}s`mxp%H?*a+rA2HhGo{{^EM2C|v!(3pL{cTG25BfvqmU4$
zJ*m=`(v}WvDPf~}*jR=wMgL(TjB+6%ODF2tyfy14G3t7%*Rrxyv$GQsbcCSFYsO8b
zaRUUkF)UJ+DoiW&5*|MYAMx*)m|8kWr@Xmy7-~IXLsd#uT11sHQR;2U(k03)RjS$9
ziJVGK4N}vVMj>EJdrqY)r79h$Qo=^{u(1qPs-*H!DeRn*a;52VHgC_RiR-gC8f*FrD(|PCwRq(uT5>vb2aS
zWu(;Gm8FZ6d9swvod~J~)qpm}^rbPvm-e7aSxQ+tkfnr;>S1FUvQ!2iQMU?{C{-uw
z*}OGtV$k&zuZ2}9n>&#}M+7RoX4+JmHZ6gw1*+1GG|^+TQX|lP3e!t5!BmW0=`ft~
z=1xD>dg6wzl&-XhD`lqC+m@xvlzFa{-JM9PB-J1#b!kA)6s##~7C^P6N>@r(I&h_g
zjp|`z8LkvldQ7m=B~b0;eWqT(g=zRoL(QC#{rE$|BXoyRcr9MOx
zAxp8OK{uV1eClr9Af58&&S9wagbh_GRcQ%T%0#KSD@&Is^HeFDJCRe#sX+?b(hxy{
zq32YpQmWFss+6!n*jR=tb;_C;j8WrBJ)4PYHgCzA8g)I@YhhK&-cCf&5rPJE8mBF-
zv*T!spl*Sx)N7bhfz3)4B&mzusKjoigK)|lJBOjxlQlG@G^IsMDFdb6jx1fE%rm8I
z>_ki@rUnryOG89+Y2RZiO({+3z?2d;s)vnbm{RYZ2sYwkh2z<8FQ4u9?`*H0KYaY)
z6i)`TSo;1s@T
zC2Uj=8_V#d68Cdllv3Mqp*m5`<}FzhgRZA|Ei6je(}@H+B2ecQ)27n2X$e#;P?T=G
z(Kg5sEF`J6UJ2*M0W=J!yt&hlwVt@4E2S$f;!2q*^>$B}!bbJ5u?$!0P$TM~jTu*}Q}t}#nl&}*daBpLvXot&h@c|`g^6O^
zR2nx8f_lG1S&I8}0j&h3u~us#1O7FRpg}t2&7H$g>j@jGQmWD-s+5URZ!?xIQRb;q
z_H-hrl2e0lw55%Hh^FUMs#2=bfhr|zR1X`=P^D5}yA&;)k>f&jGf~dwtyxo}uBUu0
z>`K|xi3~boQ0xZDys0#AS_U->bfqeo;H=kH`yee=HiX0q{9!uf&7FR%^`s4DDP?I9
zS;|PMw;f9tDf470yE+k6393O@`ciDZ1Yg>NDrG5U=|GkeHmZk>WysQ|`LIUMm&Bx=
ziE=h?$(k5+J+*6LQ_7xB1ke$HYOfeJm4;0VpmKqxbYo0VT1$=HO0^0~*^pS+FvzC7
zv2z$|Jz+ypN>N%wlrmB3?Z(n2$~;lZrcUHkaw=BU(3J+#1JLs#3|ePDIcVf-0{VH)Xp=nD)6tJZ|
zr&5(tl@3%XVWWE3ScWQQ7^QO|g
zX&KZm(UroHMgvV6y~LbSD}6$h4$~=b?(}1=Cv7N8DNBpUQbtO>9a*|anI}ux*omM@
zPz^%Smxd7GOM6hIETt?R$Wp>a^{}xJS!%KN(LtBmge*E+
zKt}{Bykgo^nl>$gswJvYZ!neAYfLS*%0=a*Piz4r2jP@Aclxo`6E}3FbfraHDKn+s
zmMmSS%yXse>_k!}sRlH(q%IBkC-$UDS4vkpaHWKe>S1FUuGF}oqSG?bRs-$mll5%g
znl&}*daBpLvXqUTh@kfoR7MXgMPDjv^ra1g+PFknY9&0WbKZL=6?V*aKC!f6kWP7X
z=P=ZI!iK7psm8Em4&!EWC19gfFDC
z7V}GejN=Fzq*LD9ISjR)u%RlYDlMW)nJD$PWa$!Ro+@Q$Cvqw|705$d8iIo@?KzdI
zl&W-~N(md)!^SdHsW;Io6HFM_shf#vHgCzA8g)I@Ygtt)*x88)IzmwICF7>jxM>ko
zFHx0h-L!U!!D7QwErXG%g$;vr%9}ffq1F>NRHamw^grG$;@VPhGpRC=eCk4`4~YHTLT*}OGtYSi_VuZ3MHJ3Em<
zM-1w`WZqPoH!XvT1-epctkE$>A-uIpM>JL%->n;_Q{LR^$68O?P?l1b7LlcllzLmT
zbdfSoma?-GL6xBDqo*$on5Y$O52}=&KO{Hnm5UBABRHZ6N8ElNo7$>cV)21a*vp`k4k-@3pjIdS&bak>ev93W5
zvMF!u^kc0jZKz7AN{gscMoPUMS-MD>r%Kt_iJ(eQ^%0FiXiF7rX%DJYrBtN@RZ7^X
z9yS)DO3{PZM&Wg$1Mx&Xo400747#4;wXiB>Undgih(I;w%h8tB*=)2WP`N}^s(sMf
z3WMpS$_8shOu5oQIOWY9>{8JM;dB}!bbJ5u?$ygqL)o$4L{y1ZZlEO=B-&%qpqiVEhBtA)X?t-TLcOvl~P(6WtCH21r@c>QY)>s
z(N;U{bVCTYS&%%hoB3>xsNfE~^n#0CX;-GL07!o7k
z1@ZHEdi$1|H-&eFw}tmb1RWu$!f!ZzX`Ssx8wAy1iLw;GwK@h|neeM@qsE@ui3l2`
zQ{LP;47HxHp(GkIz#DSZg7IH%V1GQO)KpSvQGMn=9z{{MqiRIpTkQc;3CM>+}8P_0@iN
z(!Q+D-S!0HRUYmyx5vA;
zcDtAR=gm*9h0g{`#`bjfv+SZC}3Jou53q|8OFC
zZl2k`@E~^#Zi8qKFvD#+Wq=UTj&Ift!YOa;9EMs?*3gvFlolyo7%27jW9b6r4k-WX
z^nCZ^`o&Yk2k_gs%zaa1;Lo3K&#rd3q@JFCaq;=?=O_3poIAF4BBl~kX+@eTGPrLN
zM31R7rM!VMxPe00s2(={*468+4gG$m6m#@6T7Q@r#)axcHJi6&O$@r8;sK6c@%$rK{reRPO7idcz+TIIn-hstMSki*s
zHFRPDJ_P2Sbjq7M{aEWs8_H73(h{YOJr=vG@evC2&TN5a~NtpNkdCYOIpN|GEVAk#?r;fJWI-+PJ~oKDt9Lg
z=x))9(jHQ2Noh$3mXxqjJ!~w)l46VBfGvHsh~q*vSQC
zi9M#7G?gYzOP_jyigXjOH&u+$8>i4q&RHuG_vnVfls9$yvDOne^rZBpMLa1prQVhc
zU8c&*O(juyqiBfNa+AdM%sZw@yBBzp5+4eekePc^|PNgcPDjld&!bbJ5u?$s;{R^>;
zopmPB(P=YL&gQLIQ=_h@d@bxs+0%&(I$}`k1@orTylEIz`USdD=>t;72^DNqLOP$i
zTQ^Lnyt&hlwVt%0ETt?hB1;)5vf9?wU8KyDrEKX$P$j7P;OI+ld}$A=l%QDZ4olQ;DglbE!(PQ9V>?
zkE!&e^rQn%O4z6#HkRQ@(NrT!7lazGznAG^HgCn67<4_gYhh2yZcYTy5rAqh7&euL
zO$(rMfu0npgH2z8Q%<;Ol(DIPPJ?L5n>vS~){`~#r1YdkJShVuR@MBD
zsl-$t3{`3H5~{SvRC-c+(t#%>Y*Y^$%kZSlY#xh}RoSs9ov3EBm#p_o==Bt@WksoH
zPbU)Sh(MJWOq)v6rXf%f7AQ)w!-_CcE9IQVx_h*Cl8K$NhvAerclxo`6E}3FbfraH
zDKjNh+qb&Qlv%FSv!@eDm89x}rY=RpS?JQ9ROw3TN(Zi#uu(m1EW?#rDPlxNjl|mf
zsd_eV&6*l@J=JSrS<0SHM9>j}qGt}{rqZ}+5!5bFmRhZh5hhxRW=g^+8xyPV2jP@A
zb`C?WCu?X*X-bQjQU;2zwsUnCDDzAyTRIU_iK#v)%F^H>WNDA7G^I4915--as2(H{@CcT?Xc|+%Rr1c~XB`GCo5lPB8skaA97bo*1DO))aQVFR(pra){X@Do~A(fJp
zl5`+R2^-bJ#xf*njLL?96_DeV_Y_ys>i_YCT~?QA$x-K$J33YTj9}OO$z{l)aqD
zspM221YK!>26Cb2REkoH(mSG*utC`Po0qTqi}&`Ir{~We{jhoI7rXuQi<2kY7rTd#
zA3VH$ta;?>;`;J<_s;g}IUe~KWvUIvpz$0Y+HWtP?e@11O;xjbOV-q=>#1G~t5SAz
zB7%+(6t`??ODkbm-obtxbVW{$7W6h&=G)oYym`7TIZd)7C`j^J?X|c5v+FTd1{@v
zN_m->QaZ?{ys>i_YCT~?QA$x-M3gd7>g~bOCCV&O>e$MOoJvl`W>9pc!P$W2tj*G=
zmQyK8DM|;Tl(11fY%D{RV(FtY*r7Dh^?s_F&0DgjMqN+!T3D5`l@k$kgrLqv#!aPh
z(;}!?penV(M;V-mT3D;C_9AhsZV*m+W9Kl`da{P5l%}+ZDP^G4ySHE$DDzAyTR9O^
ziK)K%hqBZ_miCxRQ%X}hFr|cz>S1FUrWDPF1u9lA$5TotirKssYhuv#)UJg+DSJ5)
zKt}*-9hNv#m4+HsX#=3aFVK^2f>tt`pn^3Pcj_?hIdQ9QkWHCm=Qak{6E+m36s1K(
zDHEmMJ}h0L%oC+-=R{5=r+V+`N`uqEnVwTAN-0VQqLi>vJ!~vQlxh>u6$Qg0`gE>Y&GQZ{rVr;<~>x3s0fDcI7UQ>jX+N(ZWxuu(m1EJKx|
ztp-|SW6L-(E>t%YjT(t4tXo|K-nh$m&9)Vsf6mnXA4sbx9a*|anWswG*omM@Q1xEYmSUHB*wP+UsY{|p!WeZL>C!0m4;0Npe`=Zl*%X+dd`Pvl+(s5xYDsH
z9Yj;!)Hw{bo~)rKr6(=nNf{^=?=IK{$~;fXu1>^MVk(-1P?ci$rKX|h0aNKo=}GT;
zQo;sdV;P=Q${3UvXh1wJR41z0yd`U5(Df9rg+(cQI*~v}1R5OErq;A+3Dhl6l%jQ?
zRYGAhfz{dz7qC5bB7g?rlsR_}W34A{=t}8Ii?~u|O1;fkx=fkpO4-wiq)JltE>f2U
ziw&53PpWjKbfp7VO4z6#HkRQ^gAqb{>{mLjRHy3MyftfT)b&)aWo4;hQzs(m2tmCI
zjGId1rbSS_Kw0WUv*}BKFU7Y0zFC2lNT5MD<&B-gQ0vJWno^q5BBqpqQg1t!E>LEf
zQp2uJ#8hG`E~AvC!CJ`D9#d&bX-WsCl(11fY%If+N`VEC(W;QxsC1&5&0Df223=3_
zT3D2_rxOWuM4)Ke#^X-s)vnbcv2%gX47cQe;<3&bTON^V#SQo`xW$hYS+S^lntE-pd$b^
z*anEIw9X!*Er6N@deRM6ovFrM7GDM|;Tl(11fY%D{R%ILK6(nK{jrBmf>-kLQv
z>Uzr8!mgA(oyedg2GwYoLtR>DpV5{<OjYO1;fkx=5L&N;P{r5mX7P-f7y>Uy4!l`>N5ZN}0?$~;xdo=yZ+
zf~t3lwltUsTiSyvRVh{JK$Q|Us)voGs8WyC(3shinAI~;&*rUJ6N9d&crC0-+0=;y
zIwDZv0@J3_v}p-cEl`yjhlTg1nNJY4)iy+#Xeub<-E7JmJN;PeNgJwCs?s8=l#xe+g&*rUJ
z6N9d&crC0-+1H5#dY?djbb)D8Y1%Xds^bDxX~1y_1dof@`^6|BTw+y&9As18*y+bw
zPuftGQk9larHqt{JF;||Je3y+NN6r~iUMMNnRrQU8VU82krrEKa%P9>*$8|g}$jiQeHL*-V
z4V0z9Sjf^IQ)xmaK_E*HgR}7NzXzL;@WV
zsB_q*ldcr4*6Ulw&=ROvpeWsVs|9u_#mXybv@@wC4Z~>4n>zhi>j@j0Qkv2trj&_N
zZ!eZEQRbOawsaz=l2g6)l%>I7L6Pk_m8O)YbYMye8`ZXZ{X-WsCl(11fY%If+
zdTnG3-XzvOZYJv4yftfT)b&)ag=HyQIuSuf2#Q-FjGId1rbSS*Kw0WA$5*(BElRb)
z5_{)lB7p|sls9$`L#-!kXi8~Hi4)RqaaIrOrU8q?FhdAwZdqw|^OiQ{LR^$68O^(3R4a7ICG_lzN-7beS^Gm9nQ3NtLAP
zE%J-HR6>{bq)JyxS2}Q|gpKN9V;Qbg3S*-X*2ZzEI#tl-Em~8fuBUu0YfB}YI*~y~
z462-G-c*`54TH+CL|cj#_)$h3wem4w-%>1X7)Q`Bp7I7yKh}EUhPITpw1_QbrqtVz
zrOT9Awp6mO6G@e%iVfQ-OoI{Fu)Zf%+EUumfh{F$R1X`=u%&2r;xM;IigBg7nW$&;
z)~u;f*HgU~mZfa$LV_7?C_$6bof75f3LJk+^!phQ{LD)
z47HxDp(&*)En-R;DD^gE=>lb*DP>2%|l=%2opZk!UM8h^D-$a~NtpSwl}sPg=y2GEnMm$I=DLJWtB5PQ+AV
zs<)D=H0S_T+G8p`DLv`HlM*(nhmB=;Qf!qH(O@GadTLA*vw17l#Gvb`T?>0s_H-hE
z-Um<@9lGaGmD(CrX#=1lF3^*1(DB|`t%X24rx=3MR*rAf4YDb7?A*rSdcuaHl%lkR
zC}pBlyuDzTDDy-qn>vwG$*JB7y3(LMTxrj#6r~iUcSR{-gRrp-Q7Wx8m{Tf~GfF4Q
z*}OGts)$}s`C8bOvaJ&tbi|;+F>h+ko0dV{0$r&=!}(}o@iD7ZSggU12~|2wr@Xn-
zkF}n(p)92=Eh0-9Dfr@5+K#1*lzFn0U7ZN31XXV$eQD4RzO)Bb%2LYGfh;9#R1X^q
zk)=^KOCXIKPwJT{XY-byOuSjf-Rj0pd$eFSmsPwT4!b30;pb~Dcwk{z1Pl2
z8>~^#rr7^w96<7JHsy_-!%*u98;Vkj(juaiiBfMTmM&3diBiFaPUKW_syFCkNmpv&
zN_$SFD5WSJh*H8v^{}xFQL38e!wNG?$93vdHJi6&O^v#q>b0;cWkV+-=mwDTdtk%oZzoYn;4#}PD0r@Xmy7-~IXLsd#uT11sHQR;2Q(k03~RmzS|
zgboKO0ALsJ@3)kNvLS8jxrHJgLKN9JBOjx6E;+(RHa2!
zDHEmMZY*7*%u}Uo>O@W@r+VXPOM}+1r9G!ol~R=sR4HMjde~ToDpgWo5kRymF{fuU
zQO@SASyQ8~r+h8!O4-+m3_4;^;~ev*(!6OI)GW}ITI^D4or)%+=a^E~SUtX5H%zCz
zxzmrep0uGXr7SHXOBpHkc4X-yWu7c$V<&e;+CYhuv#6t9I$g$_1)YXOz=Idxh;^
zkUs*G3&!~~%%;4t(~q^Dw4o}cDlMW)87cMdG1x`QJXOlJP6SngsyBwVG-wH1+Jh=p
zDOKq}l@d0phmB>ZQrE14R7xbeYD|=~c}v#BpzEn!i<(jw+17~wIs#Cvac0<58a54p
zhOj_WinRl-*(KXy@x8;pgY=2Mf`e?z96Pr$xSp_~D5WSZB1)Mk_4Z@w5@n7k#q}N+
z*^8&2{b74{y?glh!NZTw&V*DN%M7r8wG>iHB{do-Sl}GyaLA~HYIbYU$|wYfYHqMa
z+-Q?pX|0X6+G&r4yb*heNuvz91R84`JXA2z3hafgtj59xWR3Nh8WNpw(&1hm?jbh|
zZaf-aMlZbd=rD*y(bhZfeLz5oAQ4VF7|bVe!G{oH6cGm$F<8V4MLa3um_>8gI7%E8
zjsZhr1iT=A9#3!IQuC(puJE?-zR0QMRB!NIqbn6~r9G!olv0!qL@8mTde~ToD8-)2
zK^QL*8^ujkvw2I_)TryJUJI*Iwsj(cju2Ej6a}=URrVTf5!5bFm3q@uzRscKwIQO~
zb#XkYbP!H?W9Kl`da{P5luo0FDP^G4+m59RlzFCd
zrR?cM0=-Y5SR#hSX0)ZSa`oBLhCqE>peprRVdY9!*_1bS`mxrN
zHdLilr6p7;BcE0cdaxn_9!B1yHv@Q>xMM9^1Y}f&Ji|(b2lZ
z{x5@Q%9}ceq1KZ%^rZBpMLa13rQY2IyFi)eN!ieem`Y6b8f!VJO0fZx?=h90l%90p
zNeLU(!^Sc^DfS}APUqMxE+I-Ms@c3HYhuv#6t87PX<$bu66lCPy+bnsy3#6LX-lAb
zfueMyqF2H@p(6TkXzekpG`0NxZaC%5ojwTH6E}3FbfraHDKn+sW-MK%%yOlHJ)KCZ
zBvo`Fqb?1~qO)gDs&u7vr2|(=*r*;h7UD|LF*0Z!lTDo_3fjCyYpRT1Px)Hdma?f6
z8Fa*;4n1EfOzZ44+A^qEpe;o|gNW9~PDFGe#k7m
zQP1YBSrdbnziz(zIy^)cFOfQs=aCQc4{ScG5*_!IUW-L{r|>
zk%zI?6E-xZG^IsMDHEmM4F
zTK+T(^rRc?R2oFHPnYsCC}EA3XZ{X-WsCl(11fY%Ii-Vw+NpIx(?T+*CcAw`NU^x}NH_uqNRHam*$ZD>n_!o70Ub1GFSRp~&L5;m%bjfJREY%DFL7bY>QXERaG
z=B-&%qpqiXE$d1>J3Em!Wv+|ZWNmKL$4%#?atvUHg;%a(d}b|R^gR6SZ+QkW_MVcL@_Z7FT(z?Kp=s)vnb
z*isjzv0ACr#L|g+HgCb0;eWn(8I=m&*O(juyqiBfMvmM&4|sZ#cJBBzp5y;iiPK}KxC)N?9T
zDOKq}l@d0phmB>ZQnc0xL153g)Wp(>ayDp7OP@D`jUVGU$jwg|p0?|DU~g
z>AB>_&MyCoppo6cDe!*L&?AjJG0+pC2idNKREDa`NM-r=_dcg0PrOVr8u$<Z`aOcW96WKeY1}*!)a4mvsn7n^=@e47Do1bY04lPnZ|b~9
ztrInfQbg$pQA(a{+{tp}$vsg@wG)&IrADi8rAbG*UvyF`q7+g3QIrB3z{VV+G{}K7
zc`T$=Li}j%vV9lUdgeNt>tIPrtrPfJ;b#g&&8|`N!p|_FBn>7<`KMP_YYCs28|O)j
zTn2l2Ymfr9!Ds{K1pzAfEK3R3s^qw2-90#`dr=
zha@%1s-%Mnjl1{lWw!IiS}oA)tgd4%X`s*v{;c>j21nLxnl&%}v=ds=hj?o-q`_G&
z>EnD-+0=(g%V_EwJLjx*;s#TSDIH=;$&`JYSdL7&Wl95uPLL|38cjl$CizUR!%3=`
zQcP)KN`Z~-VPg(cnzQ_%(n&Q$>D4~lcV}%)U1xh8EK4bL0zoST4Z#sNo5sxxLDh`1
z)XPEVC>CGxCz-5`agCezMLPA(o%g79!Uk1}DjlLqNtAsb7CaK=o+_o#2~LGm<+(d-
zY0^$?>B*_6QdDW7N`Z~-VPg(e>TC{1QY+PmdL9?sY~Pi&HFcfsb+9U>&j|#r5LEs)
zCT=#3nBTbJcVle~U1xV4
z>`AF|0zfMOb-@ufn}*E`K;49%^x@^=eO6kfq*9d2_r^6o)fL&)H+J5m)(IO#DWY_U
zC?!$$ZDBbQ<(?>|%n43~Q={>?(xi>(hm%tgrHIl(lmZ*u!^RwCm6J1P#YY1vuWPE7*tQ_N~1Md8V$S;D#mPslJZmeG5__`mtQ}B
z{rS@$zy0{r55JV@)Hip|S?iImpsIdq$rC
zYSM(J)G2LL4A}>pTy{!ch^S3z5lx*_=UWP{lQno!Jn0ZmN}%l9#c~A7El=vHbOKYs
z)MzZKG|9aptxrtFlj2DWPYP^o4;yoMQmG!c)@T*$J@sOl?YpryhOV=_4)&xJIsu>+
zfJ((UVY6x2ya3cs=t&+VPg(YDrZS$Ta_BrYw)Ca
z(jlIdK-ss3_@uY<(1va*ajY&MIEUe1p)^VQp
zF57ouZ7k61tgeGKDP>OZ=lA?+b9mlr#+2@8N}v3x^Msl-${hx8l1*~bS$XG-ZG5K7
zACjqW=zN>BPSPMrk)$IeDRHuK1ck|IezN>X40
z*qB3-%C@SF?5p)z(u+;D@4ng?y3Xo4Sd!A>1bWnB&DkQl@^!$|n
zqEsv?mb9>>z{d8lF^46M@wul)*Eq{_wafP1SX)!q*7Ko!
zF9fwSiqfoI4#B0EoJ&besmqNobsy8IZ|=NN@M|U|UL^6AW50s1KIB*)(sS465RUwlr$1<>_26H?WdRPRSXII)QYNPJMIdoV8Bc
zAWM;@Lu4tDvTqa15h?d%DTPjeDxey5z?UX55XzmPiY!Hz7P1uB*d8_}k)`smv{q87
z(YgcK-e>#ntc@jlo#l0~Dy7m10<8$t1xwm&nl>*2brY&mm#mS;U!?OsDV4I9hr1f8
zw8*Buu_J#jM-Hx&HmFim=@3;)r0mIqfpBa0x_
zUZaqBiYp!BO39Rc-x)kI<(@00+zC>JRHJt2(xkNL
z(vwtirMS|!^v!^Rx0)GC!_UkwdadbQ8?-C0{x*V$gj%2G?Y69`%%s0o(1*)(nh
zK}}9G%2MsT$WjyK^%r@$)VrK)ElZ1Z>YF?7QR{>asuWc^M3s^#`?j(iiE>MoTFRZ^
zR5(>`P-07;*NUVr<;|(6QdDW7N`Z~-VPg(eDvz|~5JS$58~Bflb++%$+M2q~`a0N^
zQtkwURt&0xC2uy(n-_!n8C_|1N+}~Hs4*mYF*~V70+s31H+Rli>!b~`6j?e%mJ%uZ
zma-g?a!;1h?F6U-s!_|gVtncI8Kx6dk)_DeLY4v>+r!2jvNYI~q$b;^`cTiKz0USs
zSsO#w*2-or;Z!kVxYC$Ylyh3oQ>8CXMU)~+3sDMeY!4fAh|(O4
zY^(BRt077+w%NWb>!UGso$Yn7Dy7&71pOXCO%9g0*)(pR2%7SYs?<4?jE(ZwMG{D5
zx!6!AP!UdjW9L0;ovgu>VoFDtQUYb;K9(a;?wL|*oxoHuRo)&#mc|@KmY$f3DaDk2
zG^M}>urY@zbx}S4(K^xdXfLyUH`d0`b#~Xmo|IN60JH+o6bPGL!{!B`VMb4C^wW+e
zYg0Dzkgkc|)uOb>roOTB9<@%`AW9LXLqsWwvTqs7ktp{>DV!&Zje*XINr$2uC@uwetd0cF>eOK1j)OEJk
zv8vS2=>&pS2pWSSZZ?gZ7lPUuRcVsYF?6+o*NI233kG
z9imD}lzkr;JQC%WDm8RE!KrX+R0&%evyVBPoQf(%l@_WL*w`L6=1`?_wnu(aMcZoB
zcwDTreRtN@)OFU^!LF1}Cm6J1&>&~_(4~7ejlLLEP3TJH1D(7HVY1x43R!C-?FRMN
zx-y;m=FT~5owPxgB1?zJQX*yFGL|D!?#WU*od8upRjza5OXWkk_|g+pk)_DeLY4v>
z+r!2jvNU)rht8t1^`?5U&h}kd8$;LGT?d;|I-LN}3P61@gw3X5^90ZsCp4uGX>tld
z-pEecIj>As&bI>SLpJq}onzEGVS^||lnxQ4B+9;JEJvc;6Qy)I!KrX+Q~_5SvlUl*
zaw?(}QCf&nU}JmOm_w9mEeGJ8+&*la>A6~G`|hl*sq3t-gIy`5PB3W2pe`8lX4AZR
zF{qo-m3n#1-zcM!xLhUq6O*o8X&Fy_gXf&JPTXKiv86+7DVefwAPLL|3
z8kI+w3I~cXJxLW?iY+Z{DX_6UY)oQHU69+4+G$g-s*j6>w(ru~n!3*VI@p#{?F55X
z3@W`jFUwn>**MbONo?yOIeOc
zxhG5Mb^=rZ)u^OU5?`vtm!6=CEJc3I=f`;?7Mjr=L1sc-7MN3D}Jcv3v+
z5Kl^=?Ayq41j;Q>sws8?Q^8bu_6}7Vv(%;N6I1b|c+$d?0vp@I#vGnhCncWLIu{y8
zdfMA;-<7qoLa(#D4i=@^%IKHtdh
zQ|H)u&sryKP^GBSA*z%}*|(47h?IM(lv*c16;O@pSrURR6>RZ67n+sJZ6$~{#|u@j&QsLCVl*wUDuAIncr
zMU|pT3snkiY!4fgs8VIDR68{;KVEFJeK*#|&~;YV!J?E>C;0Pw{?s`bvS!n)dGe>v
z6N=J@QxZDzc`n){kCDo$Qq%ZSr$0nf-_-dwX`QISl440mSW@z2AHV(h(+|IB^)a6M
z2G2Qbow&i4VoQhEQZi-VDwZQt?%7frogh_6HROyijVXvQJxLW?iY+Z{DX_6UY)oQH
zz0?nT9ZlPoHW%8yORLz@YU(=c>sVW=D0PBCD+Y~0lQ)~@&5J?pgtjzVEf=qJNJ(Dx
zH{MD6d_$HN>C`uOh=i2b>aqBiYp!BO39Rc3t5g#x#vph
zb%InO)sQ2)G|H`_YMrdX
zlj2E-cv1po-$s@rQ0{qBik-kzFg0Y4DvimADm^h3Pl_ikJSniTJ#5V3Nww8VUhz{t
z)r;!IHrsb)Z46y!c^xcDsda)tD+0AaUV+4wsvTYFi$L{+qV$n<8j}`XdXg%x6jxffQeb0y
z*qFnWN}tBFU&A!c^jz(;eRtN@)OEJkp|aFxTAe`93PDZK#LcF0^F&aaW|XBm+o;p)
zu~H*vOSQ2LSz4r1-`shRS|@B!rKr*&s+2_8w~ggUlsl@_XG)#mR5&$ci!F^wi7h=j
z6;+BVEmSG6u{~_ep-OEsCTSaOeX8_vvCZ~fSzA-r*uG#o+_o&2~LGm
zgSB*_6QdDW7N`Z~-VPg(es;zNy1>ZG~m0ql~eRtMJW9mBV>tI((sS^xZ
zF{lcfyxBBwUJPnxbft3lOx`GUvMv}EtlVwzs{UM8##7(mIcKdCH`r2a=@45=rtDkD
za%9RqTS~7Jqzb8qEI+*wrYXjJk}9?oTUyvsU}JmOn8cR)Z0z#{sjFAj$HhY1cWG@+
zU1xnAY)h$jfEys^FviD<$Q1aM{-pq>Fg!8$9Q%b>aqF
ziY*;sOUaatYgvv=xo1mhcY;(Q)sPj!G(LYzC#hmfv85kvDX;--%wbEdHaTZ)8+RZd
z7yE4AowYS}o$Yn7ET!HF1g#J>$t7oe>7JFNF9Zz}%2I2^l3JC$yaE%V)C;;syP^B?iQKcuQ;z{wOg(n3zwug;5JgGK5%7Id;{-}?XUTm{{SJuYRb(Yt`qLfZ2
z2(%(l7Zho;Y1+I9)XgYLlW{p2xqL5Am8K-0>U5nyWj6JVopaVYX@e?7l@3v*M9RKh
zEJvi=Q>9co0jhv%NFG}nqY+zrf-0&MRa&T0U}JmOm_(IYF)7AH-Dn?n?R~cI&e|Bd
z&hk1~mD1=0fmQ@6e}j`Yo2Jc+K=p*G^dT1+tg|Xe!MzJ2Os#4Jklf1u5Ket_=bW`p
z+~7)ar9)gPnX+#i%aJMfTq&hakSe4alCTfDREsV>NflR$D=l0pu(3UC%;HLob=pfg
zbbYMnYN74Bw6>
zg7ou`dB;=V;5lcl6F1mWZ0QhNN~Y{v$Z}-LEn6Dtb%InO)sQU0G)5JpK1mf@iY+Z{
zDX_6UY|LRxlb3=GWmDr0{^Mew?YpzKrmnNS4wj`9JAt4Tg6i;mj2K_Kqc43SsGm`m
z8WWw%!8wz>yjZH_WM?f)i*)LnJMU5Jgbk_`RXRkKk|_HYvK)zWPnFW^1gFBOAsK9G
z3|VaH$*HJPRB53~fsO59V-8gsymP@R)#%lDTx_#_SJu|lb+*^Rs+49Y5VS&26%=u^
zY23UJ)Xb?$U2M7gI*sdj=>
z;Z%8i2wN&(F1GaKR8%Rdv{0qM#`dr=hbq-hNyCQp(53oJk66p*I@@<=ZB1QgeI4vd
zDR+WFzhh9Jb5P{XrgP_
zzdrw~|N8msZ-4*uzkmAc=O2Ii;m2>k@A-TF`1?P;{O9MNKmGBW{GMO5ayAAzch)#q
zda=*;-B}w$*I8Z%t5V9HAkd0H<%To1bki5~Uj!Qd*Q4gsfB*OAzy0}N{_E5KeEQSx
zpZ@mipZ=LVjo|s`A6dtak6syKi*}4z80ns8Uqv5LHT~>|4rm
zM9Mu?O1BfB3aEydv86Etv85-dqDoPvg(?L$wucQ_$^P^APk;RJO=o_;c>dcz{_^{u
zKYw|xnSc59Z=b&YYF<6QR`$4o)k|y#FG*z`_{1>
zfpW`}23noKR4_Hfger|eK9+|QQ}LvD(!!Gh8{5Oi9G+Ag`7H#a8wJD{+ic&JwJ~&^
z<#n(qrPK)mtq3$oX%4P*&!*8AfvO2b=|j#rtKfx+a_~kfP?b@QGo?j1_063#2-k@l
zTq&+}h$|&i_U&UiGUc8trPc{jg;bvx&(Wp9i7q`!6<3NYEnF$Eu{~@|;!3sFHs_RL
zy{cX*3@;@*TJ@wUMCo|Vo>izNFhwqj-ThA44UGEw$zy*|MWgQub^3%t!i9r
zkZ}5tPJMIdoV8BcAWM;@Lu4tDvTq~H5h?d%DaB5JDxfMi&+(-($OC}x1XW}yvb2z;
zz{d8lF^MddY7HGz3biY}SZDjLtc{`T?5=}NDXmTbXa%6oXToOFuz3Nfo6wYICB64H
zg%pC=(kRy&8u_C?MpNI^d5>BrYw)Ca(jlIdK-ss9dWK^bxXBT1R7z9C`Lx
z`IH+0^f8+{$IiD=>x2!W6j3@vl#(d>ma!a(a!-`f=>(_3sUaF%sT9tND?K?CQHm%n
zL@BVbJ#5S&O0_j!$81ugQR8Zz?YpzKrmnNT4tAwDe7gDRiNn@#iP#h_+JSE}V$
zsS8;^IcTrU^D=(zO3Qfa8$9Q%b>aqFiY*;tOUaad+gOfFxo1l$b%InO)etqpH2NgM
z^dwbmDYmq*rNGAaurY}(b;+jWyldQmd|WKFeV5kO)OFU^!M2oMCm8g522J^SZVzD^
zcNC^i1`T;aTWYN{&Kj#iOh#VImYS!AEG^QhZ|>;#taZ``S&A$jAxnvrjVoD>NVz9V
zX?6ls0o4!%fAFR9aD6&K6**Mb>aqBiYp!B
zO39RcJ6VoQx#vo$c7jwP)ethe^l5^8JV_N-iYqN#DX_6UY|P<>f?;ERL9`O
zkf!)dAURYjm$2l={MS!ke*OIQ=TCq9_Tx`K{8EHd-`IJNS|@8TrI^worj$V0_l3bD
zQ0|#hik-kzFf}}XLzY^Rr6;ChN-?E{DFrsRhmA>0X?z~I(v1eI-kgzO|$03pKeA?8lBHRO2Fh8qt4FdMypf&5KVnk=i8)pq6SNfB^_c(
z$&-DnSdKioXGv*vf>NQ>AfJg5rQSquPfEp-Vo3{23T$i-8*^Av9h{1?afH@0r5DR=
z-<9>zn7YpTI@pxb=>&sT3@Wc{kT;v=&5J?xgr+oRZ{+TMGG57l3el#9C@s^eZ|u$_mMXP$IsvMHY6uov8oi2MpP-5=
zMU@t+6xi4vHs(;J-b!ezXd0&UVx8@~vNndUv%3y9rBpfrpcQ~>FGu%~rD4ZUb1wk(
z6PnV8wka5Gg2`IRuTY`o#``ZtHg%4jZ!x$|*dR&~r9(t1iL&qGf=8m<6QvY7!KrX+
z2nJUgT^3h*aw?(}QCf&nU}JmOm_w9mZ=4M(na1_Ut97>T&f1!~&iXpomD1=0gH{Zx
zd?If)&6^j4ni*ZG4aU1{ZB$zN1jU%fMg*1d)His}S?k0NwiH`B#FmmN`?j$hnR3sT
zQtAY$LaHHXgsGHiiZDG%6>Uu{~_eVM~pY$3lWU9H;6V_>YT)w(ru~n!3*V
zI@p#{>;!{;&!FBn&D8$0rH>OXUEovgu>VoHaYQUYb)R+b}BZkbX;xf7TQrUsvorBV7lT{tln
zQ;I1qOewIjJ#0*3O7+tr=$w7Ms9tQdeOK1T&~=vA!J?FQCkV76(BLC!vuWDA2vkid
zN*~(glwy=7Sm7cSlJ~7srA0XP%^i6U;0VHX;s#fWD;?rW$&`JIS&mG(=St~!f>a?@
z`7w$vjn0WKJxLW;iYqN#DX_6UY|PhoDBWT~?vOHWM2lwwK?QwnTs4;yosQY{^HvZ^ZGDAu^xX8W$JjiKu-
zuY*M?%}x+#MWAxQg|yi;ZC(WGW)!7LCzG5}DG3*acK(#v)Him{S?ieL`DEl_D9D#Drlv3;jrh=)#8)Rv;
zS!C&nshCnsX<nyK>MJc^b5NJi9%16>>
z)3kXJsF_fdJ|uHYmJ_c&s+gazL}*`st1H8)bMCx@aGkipmEuZ=xKc7@-%gezQ|`G^
zs+}NJNHutkE{!&cEw5mkvE&>&67b@p3s&?7qi^Km(Oup8>?bat*>=OI`z#R@wP`2u9G&%Qe^1}
zSxTgA+{$u9$~{?1xf7rYs0Od_rO`(5r6;H&OOd4?Wht-$Y|J4`z1C4~;RjtG>$zBG
z`>w2wq3i6fgH0*jP5@{Hpve<9yN1mRK*NNl^pUjF^6-m%qZ5^@vYan%e5@<7sc-DO
zN39b!h*Cu95K&5^>|4olB+5NeO0yH33a18_ai!4)aiu4xB1#dZg(w9!wug;5M5&ym
zkrO@HyI7mjt97>T&f1!~&iXpml`5K@V9<&|qYvcGrg`&XP&=b5wMhqg`o(%1x
z&X_ua%6RG7ZNfld)EiG&*u(3UC%wbDikd0NFMhWrbVx8@~vbLtKv%C&=rL;RipcR424Rq3G
z)3gx+l~+M0bfqC=mE`KZWY4n;R%hFI1?FQm^-Y~`lh#QZG%1>Nh$baY_HAT2;^dwt
zrPv8b1yX|xm{KteVoFa)MU$dQ3rz}aY!4fAXi^>JwYa2WYTSRk*k$`}tc{`TEU$w#
zDYZ@zXhopT2hwKKw0RMzn^2P`o16(YN&l&c(%_WKjT5CsHua62GY8j68&oN(bciY?
zQugg*IU?nrDy7y5Pz6+j{2axWS|hgf1XWZis|4ij1j;>6N~;r?3Z@1p54597b+qQhR6Hr3wD6?B#`dr=hbN87
z$jYi4n)G6q?YpoxhOV=^4%MVC)93_$R{UvvAZs?wnkRpzG@&MaXm4CjE+w5@%qkn*
z@T4E3sc-6ho3u{UU`es0Lo6wIvTqH`ktcU7sr=NJAF_Y{fB)Acy149>m#C3vu$`A{H9q>}b5KDCxy~S``Q%Na5JL(%DtVz=jfbS{?;*L3{ZXBflDhPVz=jcrDEGnKu|4j0^G;(FFN*8B7W$2djeB!OAZ%
z`Gx78@NI+q3YF_K!DhKT6P$b=Hu8lSQ5vnhO|MT%#gbx43rh-YY!4fASW^3B5bssv
z-u=}s+jnDaOqEsM{xY;yrUI^+Z6s5ABT3L>>m;G3UoN{h`sQVaB
zePidHgX?4srWB_!#Ay&H`&O_VfpX83(&PlDf~m+-`Qi~x7Fl{?Dy9@uT9{H`V|&<`
z!<6bMpITLti-+~a$BS*Y@5BzT!KrX6PnSj`ZPHI$dU7hN6jfTNQeb0y*qB3=8l~iEYI)AQ(W>#d
zSZDk0tgWf*tgmBTX`Db6e+DnLuc%VDs(@;+0b3f4
z6N@M|U|UMB
z6AW50sP&$_*)(rn460|erE>8;M6aUsI$62ckbG=>uG8|s>={pegXf&JPTXKiv86+7
zDVdVDEwDZ^<(@62*$GmGRD*R0(`a-w<|I{YDYmq*rNGAaurY@%O+JU1y=}?Ti+#55
z&iZIfU1xh8D@!BAP9SK7pvKE18~9SUqc43Ts818hQm?hqO8%M%7HN4-H@80572(u3
zcHX1b$r?;4rgVrYB~a3~{nbaH+%lz+UMDaWObyl|OQTUDOHWM2lwwK?QwnTs4;yos
z(rA)$A^AoF@x?ORcVle~U1xV4>`5th0zfMO6%9q$Y#KH%0QD1kQY#PWW}lthzLz4u
zEJrIE#ZE;u^-Z1ksCBXiPl_iU;zP3~Fn2IOG
zlNO#7*w`L6=J2FizG|5uPo^}Uio4im`>w2wq3bNKgGDK|P7r8CpvrsFX4ABJ5vZ9^
zl!owBq|saHb4p6?-{;1ahBBM_#?CowowPxfqDqITQX*yFK9(a=?x|90od8upRhDjS
zY1Bz<=?SW+QdDW7N`Z~-VPg(e>b%XyXWKZ^bFt3$U0EAL*V$bMn^Ibx0MPFN)Mf7p
zn@z*!37{@dXi6V)**=-1Q`RoI=Oz5snbIPg`o_+C)H-2(8J5`0dA^e)#2Lo$b4_HioXVyAC#`6gmN*6@UgWJ^@)O#XppMDO>LwAL@!^>Ki)mQR_qvniNesM3a&y`xdbrd2&ya(&q%FLaD(-
zOlj0kOzBCfXi_w3p-F*_?O|gMO=`8#-niByrR`m|@4{NoTxW9~EJ-PI0zWJK^m4fw
zPr7Hp=o3F}oKcd7EHET5lUgf1PR2VIQ~i~$NT$B2^B%QM*q}+#q(d|*iL&p*f=8m<
z)1-7c!KrX+Fac8<xqar7+Ys(&hxG!l}V{Y-!X+Z0X6Vs8UpEp-O>`?O|gMRce$MQ|e>uOwZ$D
zo$b4`wx+JLz7BS!)H%VR6@ywivxhF-vt#tdpn5`AnpJc;=U4bx6^&{Y3YO{AH+Rli
z>!b~`6j?e%mJ%s>+uG_QQtru83Y`E|Ks6YLFBLT?zVrlDWGS+=kfp%J_OLOBEY(pu
zHN3N}XX4uHY~Pi&u|u!3yN)%bo<1i4v;t7M-AveQ8a7V=HEBXqDz8&1lVjFF9tiOA
zsV?XGP-ziOeN*Q>YMrdXlj2E-cv1po-x`)9P;PlrPn#2%3Z@2QQKeBU5rZeD;z{wO
zg(n3zwug;5JgNMwmI5VZO=C}OFSDID)_UkVyX#<2N}Ce^S^=o`j|4ZgB+5NeN}m&)3a18RaHUaY
zaiu4xB1#dZg(w9!wug;5M5#AY=G1>{gU(5Xl(MZoX%S9+W9L0;ovgu>VoFDtQUYb;
zB9?Ebw8=I~)mU~~i!rTDX&Fv^bLX73PTb&1
zaiv3CDVefw56h7$_gpD;PLL|3Dqo4wrBMaZr6;N4N^zxyD+M;ThmBcWsY`O^Ea=v;
z(u;+*@6y^@qt{tq$J$ayqZ15TF=+IbyxBBwUJPm{w52JyWOMX#s=!LlSYsQSIXDsXFRxu;5LbpliY)u1D`G%9Iv_9v*K
zN>QbSDg`#ShmAQ@sh4Hd>7-M=smcR#t#!8V%GwyZ&h9$clv3&hfK~wNy(Mfm4Vx!`
zsyLx3jnP!6I1b|c+$d?0vp@I#vGnBJL9#{(bS&wVwvr`u{MUTv%3!Vq;xs~pcQ~R
zZwZ@C!{!B`ZbDBgZ3Zf(q@279VI_j($*wwoifHPaI`2{IWDTAaPddbt5-9tYu^fSN
z&y&*W1g3(iL3>nbl#k`go|uX!#gi7E6xi4vHs_@uY<(1va*ajY&Ld4C+~`VHyW|F1Fde
zD{EuuI?L-=QEF*(fPM7>73JqqV$onyyEX%_VSG`8W)}5vizL?
z`svHBpTGY6>5t!j{ON~ZeQdDf{-Y9GP;BA)sNkABZuCvLE%*wP`kluX&Tisi_Z
zd$yEDCrA}i4cZ_~gWM`o>LgWcDYmq*rNGAaurY}()!E5iNaGs^dma}HZQrG}HFcf!
zb+9d^)CmTy7*u&n-fWsTF9tO;+EOczRK=W>mGV=Yyiu(P(nUP=4W4t>I&p(7#g-1S
zrDV#!g)B#=+_R;!^-kDw-dOWbT4H%|mjc|uugoXSbcPvulW)J{1Y8ZVX>;nX*F
z-lNvZ8cZpsbc87-P&Te)IRfRLDW%y7Oa)VeR>;zjW60^mR7@$R^rI;SHh_&eOzHE@
z6s@B*w!VdbvCZ~fSsO#wSzZT=QmUOG(279i$R4hA)3(tUfrbf1=_5vGRZ=**Mb>aqBiYp!BO39RcYgvv=x#vo0cY;(Q)gb(XF3mx7=}D@%Qe0`_
zN`Z~-VPg_ks%?zkhtzPTR|{?5rL{G6o%MCBEj9Ez!Jrj`MsLWQP4ni(pms)E>g4WO
za5l-Ox+Ec_bjl4|TEtV|;L-0{>%Yr((DAM!l^;Y14Ot|E3Wk9R75GFv=F7h#`dr=hbR?CDv!lIA5CeUDs8W`eRtN@
z61~p)I@py`?F55X4C=fgZ#K=F7lXPfU8zf=aZQk`3t2rs)}=-SmGRU!c+Oet#0|C-
zTROy+k}3PvvK*Om&z92e1gS!*L3xB}$R=cak}9?oTUyvsU}JmOn8cQ5SyjC}8rQf1
z`Ec!pw(ru~n!3*VI@p%d?*xNZ3~Ie0Z#K=F7lY~rP$ITwvOhc9kGbgEHOR=SeEd@5VhmARG
zsjRH#c^T5idR2W~EVO->*4ET@*4MGNRMYPSgH{Y`ydiHk&6_8KhBToq_41WYWs#&t
z8z)6i*)`sQDbuNM?wqsMNgHG-vUG?nB~te7WjP|{mMqoOI{~VIYLF+0@TFOaFFipO
zS&A$zWGS$*J#5S&OSSZC1RI=doa?#RXZ!A~jiKu-uY*-7prGa-`sfz;W}Z1Dn*r!
zP^Bcw#;q(zqTExZlsmzxaH`M`wlt(Dw)EswR4Jm+X_UKw2y#?*DT*TJfkZYL15LeS)in_c7Pg`iqu#tO`T)sorCM74XPAXIz*KcDf{-Z9FcNMl`3kT098QsxpreqlNVchf-0&MRa&T0
zU}JmOm_(HZlcM#$)j%ur%=bg14y3X=CSd~)i1c6or8oVZLHcgusfvO2r>BB3p
zoi@?i6r+?1#%2H&;nX>I-m}(;8(b-_bcib@Q}!)nIWpy*E2Y;7QiW7~ew|W6mnMIb
zDy|e)TDVeRV|&<`#FeTTvzKE%t;b3)7TUf`Yio^OXMG)POKEn3K`RFJUXwSQ=FO8q
zU7XRDM)~b2N|jG3JGqS?S~nYtcNOp#7~b4R~tt&=v$Qe^26SxThrTgq}o
z$~{?1w-cZWsQT>jr6C#dr6;H&OOd68ECn{UhmAR8sd3t7)#}w~FSC6Y*2V_C&gMGU
zlhW%1epdJ?Z$c6^n?}tGKlO~B)X0%ix%Dnz=|YfRLFF1>>OKTh-^@7%t&=ktQVi)3
zLrR$JTgGyP$vs0#rxS<@qWUbKiIJpQB;m;BW*9T
zefQN^4_#+>9qLGJrqT%jtpL>cXLSymbVp741W=nMbfga-vNuXD-Um5Kpygv+t5dMZ
zroOTB9<@%`AW9LXLqsWwvTqg3ktlaWsm(Mx!KrYnm%B8$(xim{PEJLXB1#KU3T$i-
z8*_-#=ws3sZ@`>D+1MClQx^C&5J<&gr-#b?wv6{N&I9f
z6ja{C`qur&Z0egj-zKe-G-y&Z=@3myob21ea>U6!O-h*)kP4*wY%rxE=8)70sc2F(
zX`xAhjqPD$4oxa|9-W*w3$07{7rSiVjkPg!o#l0~CZ);=0<8#C8pcVRP1EK@pk_i%
z8nTrN!4$O9>iM5cYG~3joBGDiIcuG?L6xFPhp18_W#0;xBU0|EQkt9qRY28ejV%o^
zi7h=r6;+BVEmSG6u{~@|qDrO1DdwDX<9)b`eYWq;+8Dad@;X?R(&Geyeovq(drjJG
znl?`YHF-i+nq17rWoMrCNlMQ9n)*QLvtICwroO3j&RQpIFr}E%5vG(x+4yO}BT?>|
zQfi#wR5;aVg)9v*iYz@j6;p~S{b)*o4PavuQyP>`(xj19y{fj?*}f}lJ#(Gab+9L;
z#|i$d_%nI3X4kBF@n@LOlRo6;y%gQc_qk-P)lr(1QXM~X)8Ruj^^Ki#);e*6DaDiy
zF{Nb6zCA2Qrra~7)Hy+_kg87^SsG#xS$dKxrW8|Jm{MS4d)S!6luDs9q?
zq3yf0wx+JLzK*q}iAE_R
zJLjx*(gsQlm(hUmqYo}h{>MV1z_6xi4vHs+9}
zI!LFJRVp-A)r)<$@6Os7y3X=CSd~)g1c6or8oVNHHcgusfvO2r>GO%wkb_EAI}@b<
z)i&QM{g_RCW9Q7lbQbS
zDg`#ShmA>8X;iYZdMEFN$e+;HPhWoh{PpKgfBg32Pe1%}vCsD1SsO#wSzZULQfi$b
z(2794SES9RY4apd9Vb+!T3)D<=2Lm7%7&1m&aQE!b~;6jeGzl@clY
z_OTq1a!-{~>jbC*sy+p5X^1vNcY-Ra6jfTNQeb0y*qB3=8uh$-=;YU9tx7NU+1@*A
zwM4J8ybe~S)H*?+6@fahNSjU5=0%`xLRFeowkjogsLBR;uv99Z8nuFDHua62bJjX(
zgDORp4pF5<%D#mxN2J_SrSv)hs(`9b9$Ols5ePg%6;+BVEmSG6u{~_ep-Qt?Dn>a|
zQ=8I@b++%y+8Dad?mF0%QtSkPRsd?fB5XDdn-_rU2~DY7y0^}$7__x=|K3{f8~O7w
znmVV>w^8e44W1NFI>eI_DErp29D#DrlhW!0rh=(HIaFzgT2$$Ysd!R6Y2itMjqPD$
z4o{jww%%xM>OJ*hneDr=HioXVyN>mwkxC~3v;t7$6=Ab!*gOGLr3pRhBP6BeE`!eU
zS4KPKtZBUdQe;!#*m;jyCu|U
z@=m?@(i2pXrO47kmI52w!^R}C)J9obz5MN2%hHQ|w(ri`7`o2#I#`v`=>&mR1gg9u
zZ8lAt7lE1yRp}#Zm0faH-Yd;U8=K^8f&82=O9gciPJMIdoV8Be;7W0&LtH7DvTqy9
zktz3FDWy)3Dx~U@Mwf;l@73p%RB@%a(!!Mj8{5Oi9In&{Bj-xBa`mQqwa@n5SzA-r
z**f}knq31z9ZI;UvlT&Y#^Td*$H2~>nr-`IJNS|@8T
zrI^wIrj$V0`GvtFQ0|#hI-S5&Fx4l8ER|YMk)#5U|EvG{`Ri|g|MS0p`s?Q(f0DyG$M1XooJI?2*vbIKkaay^4pM;j>$NYDPN{ejj8#`wXu9G&X
zQdH>>RZ67n+sAT5$~{#|trMUMsQQ?(rSbri*wPbJQKhKTLX`p=+rx&eWdHg5r$7Gq
zrZc}^Jpb(gY%N?Y`a?@A0xsv1Si7uBvouFwzRONz{d8l
zF^MgeW+!neF|};z#X{S6X?--NuCu-lwxyIi!Jrj`dg&TRnC@9P`ee`)C$y!~>twWa
zj>QdO1nW)(`YF_y1
zCiJ8x$`yR4b(ZH!b+RfbUwhII!PGZ%jzR0>42Bd#I>e9?Ci|AL9AR?LkkaV{qJpSC
zdL(I(%dEkkh>9V_kQRm%*w`L6<}jp!+VV+DZa_99>BT17cVTS|U1xb6EJ^8ffYW;jEPF{?w&+}aDvz>R=dWl|V
zc^#`tJ)KSvXhopXFizTRnl?`YjcGzvDvwdik&5iBlx-PhZHh)U0;mY5&bjlRwNBjN
zN^zw_Tq&8dZyC#xDYsmyr_%{ig;agC=+Y3B=+cu^aizG@!j%FW+r!2rt~AStI_-rxOfXF{pO(wg$p9?kG%O4C-gJrE=O?-lL8wg{-2oIi|+V
z23^Ec-{3iCtrIucQf%oETS}(vTgGx^$~{|3rxTV3ybm>W|xKdo{M^_4L02_0-((Jv|4_lRLReH70_T5=qQ`gyE2g_2roj}kE
zL6akHc8!}Cf`$oY>GLc>mgl^DVd-f7x1t4
z*}gk#Yw9}N>tI<*u@eYdA!u-kxY;yrUI?ltl%>&$EY;2ir*gJg5~%U9t_Y{TvGX3a
zPS#*bF{MLHDS@(Y9m^3Y_e?3RPGBmS>O(}9%Hw}QE)+dqih5xxrW8|Jm{MS4d)S!6
zlxpMT27dIpQL1sV&Gueds}*{k<#n(qrPK)mtq4@EIFmM;rp=Q;eVkB~I_rY-+QeXt
z3OQ=iI#gO@Q{UJ*b8wxsL6xFPhp18_W#2xQBU0|EQfi$5RY3K5^&DIJZ1i*|sG>?y
zrG+X5HnxY2SyZXf+D4}utLnu*+jnPe3|(h=9jr>Jb%H=E0(CBtHk+o+i$L9ksx;;p
zk}=U*n{84-oBE+rZ9Ya*-_$v0trIqwQcUR(Q%a)j+s1Mv$~{v`sS}(Er^>x^WU2Ie
z25(PJ#gt-73sVYgY!4fAm{RGUOG*cm>rJ)2&Gy|`>zV6pu7foxg-+mSg`d_XqGr>m
zdEuv?P?K8uB`JC|Wv`y$<5O;=PZ3OgL+3qeovgua=@3myp6pw~
za^%S^O={_Kf>NPW|KuM`sR~}7l!_)rlNOp3*w`L6=Fp@%$_pVmr{)WB&1JUl##%4X
z>#VMWEh$A#@Mp!J+9k4P)2w;%r=QT0M!8TQbBIwrA1IYF!PfiOWis_noo^|)PTC+!
z5v4;!DUq^o2g?yD_e3dGPJk+)>Vq6l!jbQG$-CUR1Bv*}gk#Yw9}d>tI((of8aNF=%q+&8~U#V$d+5
zD|ISF7n}`V;>QFl*Ba^sD$}WN?wqsMNgHG-vUG?nB~tc%U+{>Od$N>5CqNZY^*-ZE
zgWM=`@dQ<5DYCSXrNGAaurY}&P1Zyu50y49;9u;seRtNz&~=vAv8vS2=mddQ1R9+j
z+{2b`X-i)OY9~~s&$qFYkIp>noRYVV>-TaX_CqrD4V`n=I$47#MU)N^r3A{p?+YG*
za!ZsNDxJVoFxC5nD}8oaxo~1Cq7+eDh*DrO>%aYM2ftom@DwP(eXkE~9<=s1*eAcZarA0XP
z%^m%owNBjNN^zw_Tq&8dZxzdtDfe6{jZTm%r0Tugazd9{(WNJ;;!1I)g)0R%wug;5
zTxpbL)h4SOr+OY2`)uExwKa8}?RBs$rPB!ntq@e|#fh6uOygr+5l($$=RInjtihCGN{5(I0%hMWmLpK^nNlj9z*I2RKYv4(8hNe$#8gZv
zrnE4nz{d8lF^4Jja;isrIZ{&}D!o`{`);g_q3i6fgFPvIP5@{Hpw>mgX49~F0jQqP
zlgepQQIirp$)@a+6H{8}PZ3RhQ|CQuovgu=;z@^iQUYb)8kQqa?s-z$oWN8t)q963
z4NeDVPE5s<;zuB8k@ASIX6DkmD$uccFtMrqz$SRRXRkK5-Ix@u^f?d
zOO+4ph5BT?>|QYxL`R5(@6C?QLO%|VW7JwNHcI2BWhDJ@JXu(3UC%wbA%^h&4H
zx_sZ>X8Ufe^~`lP*TI^UJ}2i9WA1HlXth0T0*4ET@*4M$VlsYFE^m_&ka$KCe*)(sS465>st~BUm
z
zbb>0f6j}OFmI52V#w4;d%MU6q-PpDvOWXTw-<|bQ4_#+@9jr=ebb>%D0!@yz*)?rm
z1RADPrAliP8QiW9IUJbf5SQo56
zNflR$D=l0pu(3UCOyWuod8up
zRep@(OM|uIOHWWmmLf|FSqf}y4;!<{(qxlUu2t`JvCsD1SsO#wSzZULQp%km(279a
z6UeZoTUL&~2-HoeN<+#j$gxo)y@E0MB)>Jzm8y@~)HinYd)7K>gDORp4pF5<%D$~E
zN2J_SrIb4Xs(`AOmk6N?x&P+4j+-A*8Ag`h^>1HzZ?`GWovK~tJhmb#>qGqNq%
z;Eb{Ic4_^wt_Y{TvGX3aPS#*bF{MLHDS@(YCCd>gcTB0tG&_N*V5)Z(SsJW#OY0L;
zF{PN&!ju9V+r!2jrc`@5Q0j9|re0JZ?QOR2%GwyZ&hk1~l+x@3fmQ^nT_9~XO`8{i
z`WZ!OGCpM~b&~w?)*79YtplhGr@py!&RQpKaHY7?A+D56*|(DA$dr4olx8PL6;k!j
zPsQlcEV}e0Ra_~qv~Z=s#`dr=hbuKfDm7xvjZO7&vCsD1SzA-r*KXOhJ~L?@v(~buCMabn2Ts?@{Z74XPAXIz*L{DEk(&9EoyI
zmD1}3r^2aT-XOx3niNbvITclkDlJqgu(3UC%%Mu9fLKIn){Up)9v9ne-<7pBb)D^X
zuqvh42?YHfL49_CxY;yro(LNAgsN2ZsGKJa#>l}^=Ti)g04lPnZ|b~9trInfQbg$p
zQA(a{T*q?c$vsg@s}qz8rFy4urNP8t(n+a^Qbg%TQ3`AT8*_+KZT<5OWHgOKrR`<5
z^Tt{&(Ce(OgDokYPVi^NpGh9qK$Py;Gy3AsFrg)NR<2xy@+QrG5O|$03pLRk~`VdX(jlAs=ycAAJ
z{^ZbjC;MYG^-Y~`lh%nEEGd?Bh$SUY_I+6J$dg-^G*RaSr9!FR%6%F{sTWatQYw}d
zOIlb`U}JmOn8cFGbJ8j#+qm|4wafP1SX)!q*@g&AN|_Uw3Z}}7cgWIb
z)vWO+reaDlrG+U4HnxY2IZSCxLD`sMeV+7Um+iZ-HioXVx(?Q)G&#Ya6@SX}k7UiJ
zS@Y!25GT~65AWs9nTjzcm0V8ppHeG;&S>hLI{%!sPSjvYv7|#RDS5JQ0n3pm_be$r
zPEab8Dvz@xN`sLn>fK4HSW+x$VM&3F?O|gMOX_mW$~7+CUv08|_tn_NMUo;(3rPxWY!4fANK#o*<=&&uI@e3;<6@QV
zyRo*WuCu-l_N4SU!Jrj`T6vliRk~-h=!-%1gq~DzDCiXA-hGy{q$=th>I5p&sc-I_
zv(`x)WGS+Ah%6;i_I+3Ih?IM>lpZHQ6;So^a}-}{wD{5!RFS2~(n6L38{5OiB(gLn
zleJcj#^H-~w(rW?7`o2xI@Xj%iktw@3P6qE7_xNFYSAZv`ZS>_eFUGR>eR|f=b)9f
zLd%UOvgP^GGn@Lx&U@54VS^||lnxQ4B+9-WEJvc;5~YzUCpZ;O_15A_gM2bK`s7qZ
zDWbFxrNGAaurY@yP0FgEl5HI-z1U{^uB@#cdY$ccuqvg>2?VVWR6D5@$Chr{FZx1I
zKcOnMQQpmUQMnYoPxAarY9vq*PJLtNJ!+k-!IWZ3hnP|VW#0~#BT(*{QmUN5R4~x2!W6j3@vl#(d>*03Cj
za!-`f<^-q0sorW_>GKIDdHU~p_36c_h*CspAxeRb?O|gMQR+7O
z>x2!e6jeGxm69kMKQ4GA$~{#|rxTnCr+TZfr9nrrr6;GNN>QaBRVlCmY|NobwKrON
zHKJ-Ps*j6xw(ri`n!3*VI@p!c=>&sT44NEyvuobG7&Of2N<*^FWc7L&LOJ;#%G9p3
zjHkZAbIw{PZm^};(jm5#Oxd@M<;awKwvJcleq}flT@*#*wVt50vp@I
z#vHa(N+LadJ)HinCqt?k9Oev;xh$$sd_N`+%0_Bz|4YWFesbH!%30WGn7g>5@
zDy9@uT9{H`V|&<`#FUz7b(Cg}dQZJrX8UfejiKx8u7f=(olXE~1)#w>!e-O3c>$=J
z(34tusZ=}XgVn~Fq@AkcCw@$(zM=CTwNBKaNztT3G%0zqZx_pvC-*cdl}=D9lqzoy
zVM>Jo#gv|uiY7&q7Mc{;*d8|K(4;wP<+L-6tB>tnw(r7P&s=A79V|&HbOJvs{Pa%B
za`2>fM^F01Pa7wcq>reS(2#ODTPqK#Dw7+{_eC=GO`Z3sb;1TsiY6VRNlBD_n^=xS
zxu;1fbb?diRBr;NG-xZP^yE}DDVnsg3oSwsg;y(HDZc300|aNo6Bn=zKKJ%CAs4RUkj+zkd4i>*udOfBNIM
zAAkDcmm-|{#?E`xI$47$#gq;)r3A{pMJz|4+%u*0If1EQsyEVOi7b^~PkUl2rW8|J
zm{MS4d)S!6lxpQ;4%WykA@!o#-ex_v?5T^P^8VKY4ajbJ)tPI
z(I^v=S0S5he8@t}RsNLO)Him{S?i?yrG+X5HnxWiS;_wM_fLQP@$aAh`dLoU{PQAKmPLjpFe+j?zw;b<=4M``udMA|9^knpT7S6pZ|CN@~_YTlfQoc`rF_C{O!N{
z+wXh*m;L=8U;gv+w{tr#7;UXw(5*Msi*>f|%GwyZ&h9$alzOV10MH6RjeDNiLzeC+
zOP>I0(uAh;;hfS*{*p+SlgTa_lNyhd7TMG{cHX1b2^&NyqI8HTB~kV*VL1}zmMHaf
zIl-xLsy7x_8ng;>q4mwFh*CspAxeRb?O{V!vI#_~mhxda)?-_38dvLV-<`EJb)EHf
zuq&m`2?nhgR9bV$n@#iP#h`vhS1PY#>!eNg-g=b;mtCqq)|K(pH+arj>%(}ls@@ocX;9MasZLVGmSRf_TMBG!4;!=CQmcZu);4b7KQ0#9
zzDsLs>N@M|U|UMB6AW50sB%);hA=fd3ey*Zni*}W94oasN%B}PSFnOyGihDIk00Zy
zZ}3O~))9s4#0|C-TROy+k}3PXGI(UlJzGk%6Ql~MdZQ7hK|YqNe3B}*6kA%@Qeb0y
z*qFnXChv1LE;i2fJTCUxzB_Ac>N?x&U|CAD6A1b}g4!%Mp7Eu7){Q<9RObn0Y1BqL
zE!|E^Ss!E8zVQr95l($$=RInjtihCGN=KMd0%hYz29H3wXG&>y0#m_MZxpgLs3@}Z
z#8gZvru3sJ1vY??}D<8{3)WTZ|b~9t&=r)QatGpPfDQd+sASQ$~{j?trM6Erh1)G
zr2>GWN>5D1lj2DWPYP^o4;yoMQm?E@AvCT&UhJ}c7uLqmbynB0n$*$i1bqHHf6iYh9l9DI;ma!apa?6rB
zI-Q_YDAntPC=JSsC_O0^ONu2eEGe+DJ#5TjNnN&*+M(5_akb0#-B?>w*V$eNi&83`
zK+p<7gR{iVrg8H^P&J__eJaxD>r}=E`A8R%T)(gHGZf*}H+J5m*2x-7DW-IYDJ4+$
zEn+zW<(?^}&k0NgQ@xJJ(xBunkv}mNQ;I1qOewIjJ#5TjN>g(3I9yJ?-cv7@*}fZV
zW9T}&>tIhxnG*n70jPJDu-PHTPRgh{e~M`8n>z1N>tqd{
z6i+(DlM*QVRE@T9=T_OLOBCpAj8)GWUo
z>pk^ineDr=HioXVyAJlG^f&>a6@WTt37bvB<^`Z`LQm?wO)2V-Qi#$upN(lW2^P`R
zId#6J;5u1@C&iNv@uURGz6C5tpxpDM^f-a3V5-+1RT`8LReEA7o)k}7cv4_vd)S!6
zlj*}nX8W$JjiKu-uY*M?MNSZCMWEJM(q_}Nc@e0dP?UzCU5?UuAB{>j
z$|d~zJ6#z~eN*S0wNBVzN-?EFOeu-7Zw1SdDECY$O-^tsoa(hhmIkFomY$r7DaDi)
zrWDxN9yaDMrA8VxY?fP)jmqKnI@@<;t!J*Yx{mdvmL@0ov*J(VELpQ@)(C%woF??7
zR%#4fOj(Xp$i*uk#h2Eiv`nVHsdLU+Cv6a=h|(dVlt|gPh2@BpTcXrb<^-q$s$N@M
zX;4aB=?SWcQbcJXN`Z~-VPg(as-+Oy7;Rdj^kSdwyR$ZiuCu%jR;6?~L7)|ZYWW)+
zTe@e#=!-!8gsSwBjh1WoR-Wl{DaR!Ls8wm1O?_kMoV8Bcph{7tLsTh|vTqH`5h?dn
zDQ!-GDxm7M!It`*#g?9+iYi5w7OE83*d8_}QKcrk7@QPi*Q@HqKHGO^Z46y!c^#}u
zsdIurD*{!{k~W*B&5JtIz%qZ0)BJ%NgDvZT$XY4aq|lxI|>!N|3en4OXb0wjH`ZJjDDvZ-(E==ZF3
z(gszEDjlIpiIk1YSdK`!r%LH`0#pH2uNAh`=P0)H1XWZis`R5O1vY?%Ek
zU|i!=X?vgTyR$y(q3bNKgH8bp_#q>3xWl@_iP*w`L6=5VDh=4_PBq25#<7yE4AowYS}o$Yn3EHxB6fuI$F
zMrVkdP2=W;pmsu88dXR>WoM$dN)kxN#uG3_IQ5O4_o#KU22+YD9b!relzsbHjzGC(
zN)5G6U@DmERYI2f?7hf`=k=!-reaDlrG+U4HnxY2IZUb6QF)WJX{geRZMN^q+8Dad
z@;X?QQtJePRsx-y&k#?CowowPxf
zqDqITQX*yFK9(a=?x|90od8upRlo;Z>a!DDdV(sd6jfTNQeb0y*qB6>%F!MLxU$Hb<-E-j!9+
z=wN(Mjr=L1sc-7MN3D}Jcv3v+5Kl^=?EAXl5h(XODVFU^}J1k
zD-~CIaw?(}QCf&nU}JmOm_w90?UQj)x%%qk)i&FAWo=DeXL}v1N;QQ}AZUf4#u?&f
z)3|veXh;*P(wsx|F=p*eu*PWD>T~*-O?^}69fRvc4WblLIz*I`C;Jw$9C>m}lxq5%
zpj0T;D|w(DS1R|4^hv3RQbcJXN`Z~-VPg(as=bnb2Gx3`w7tys-B|0H>#VMWEh%kI
z@Mp!J+8MHD)2w;%r=QT02B)OTAekh;IFsDHGOF=Jc9~3lQ|FwuPTC+!5v4;!DUq^o
z4a*TJ_e3dePJk+)`h1!lSDLcV>I79pDWbFxrNGAaurY}!HPP7Yw2k$V(u;kz@6Os7
zy3X=CSe4S|1c6ors+^qH!7``9srD2GgI
zRa$0K-`F{4t&=vWQdH>(RZ666{JP)~Dfd(Oi_!_Is8UqvM^y@J
z02_0tQvIx+lS5~rv8wv^KHGO^Z46y!c^#}uX>@`>D*{c9wAnRnUIZE@RHe?En6z`&
zW-ZS{Bx@UmPGvUrjh%DWI%$I{MU@Uwr9{fU?+YH0a!-{~=>(_(sxD`2sZT*{=?SW+
zQdDW7N`Z~-VPg(e>SR;ZG24bIy;x`auB?rr>+G&$O{t>L2>`7CG&)V#Y#KH%0JRgE
z(#NyR$!hodN~cm3neXdUrA0P%j-78YxK7w0N)e?)L@9}~Zx_pvD7QqZqS6UYg;QNl
zxKf|IxYCnT5v7RILX-j<+r!2jqEuGY9JG@XzIsu;T4($2tgWf*tgnMzDYZ^8XvLsG
zK82u5_beKHF{ql*mBt|7#8i^zdlhsv#-_%%x-y;m=FT~5owPxgB1?zJQX*yF7Y2_=
zxhG4hbpliYRhJ{a)F&st^aNF8DYCSXrNGAaurY@$HQJ~wua(BeTcsEKY~P)=F?5~f
zb+9U>)(HZw2-M4ikl4~at45y$>f(f|G)ZGqvMDQhnp%02yf*d6x-y&k#?CowowPxf
zqDqITQX)lf+g^P{$~{#|trMUMsLC-VY^hIHZ0QNAs8UpEp-O>`?O|gMRr>$2_clFy
z*`1Sp#pWl7_?Z=;f_~jf1Yo55lmEuafxKc7@
z-9eT;Q|`G^cAX$qNY!PJF7?TXEG<>w&?(>2#dpA4!;w57&H?TvHF#5xEyb3eY$>p@JZubNOP!Hz
zr<8LptLo#T(6%kDwNvM*uU&1a&a6AZpap}thH%)S$(3aPqm5vD#V5vB*JVoR~5
zCtC_^EDsxF*i!jbshmkURAp(k&$jKXB1?-?=V`BJOoSu}2*2HBZ)HN-?EfOeuk~?kdY3DECY$<4#~InCh}r
zU1Vt%S$be9rW8|pGNr)A@~|<6Db>aY`3T;t$`#0qHruwcRt}w~yml6)j5|S~1%WE3
zNt;E}=1HJtKv5b*l1fRC&q{-GNxSGO`E$;uwy|@ZHBZ{0N>QaE%ovWV9W=oqDoPvCshh;EDsxls8T7bUdvl;l?O{N`fS_IS~+x{^4eLI
zGVTO{eode%J5Aawnl>AOnmnK?)!HU&jQrY+vOFet8hlNap0cTJ>}=BZa}LgvHmFim
zX%AIOq%1tjvPa52Rm!##pbDruxlw~HjZtjr0jj7{ROwq)3TyxygQ!v^AM|)>s+9})
z7k##IXDv(gJms~sDrMOT0xbwMInrj;w0RO}7*Lg(7)0{ekhf*0?2&R$l``xEr~;}kWo)UK2Or}Bs;E*_=}DCW8_UDS7^+lW
zDz#3@75vJu(?y?c+gU4z&Qo6ds?x-)69ifiXq4aH*wQt}MxO*~2UMjG>qYWpYqWi?
z3d&QZm072AI5p0lR}jt=P?X0y^=V`BDy9@udNQTJ#`3T+hAG|VEUfcJmp%2O%(iW;l|$$0uAMz8t4;uD
z0ifO~!e-I1*#K0>0X=Ce#`3T+h$qc1BpZ}AWl_Covu!JD<0H|s)TeA}8#~8Y^P~-`6jj(_(sxEnKsgFW|_5f8>DXR3ON`Z~#VPgO4)P*Q~^~d&ktcsqZV6wfGVmKReDmTz{c{hF^DQv
zMs5Kl8!M9<7k##EXRRDMPkHUDN+X+25NJW5#%-%@*wPhk=|-R`4X8@xr*l3>6@o;M
zN=7~(t!xVFQ#Q4Yo#U)|(gszED(#|5iIjD>SoTP{rAi}{PJk+)>XOBl`lx)=2dJV-
zQKctU3T!M78)K-_Y;!cpr^-h?7j?F6Wvv`KPj~HXN*QzlKnnns>&=ACqG9s{P(Pq4
zwK~WbrCKX%ym(TZQ)N={6isbY=XKOPS%W9VlXmf>1j@QgEPJ5b^P~(qfvI4sO9oZy
zLl#wfU@D#zPkQpCz{c{hF@`7g!E2jLsC-d+QD)mV*2xFG
zJW+!w#guk1rR2%VZwu~ua?g}9=me!gsV*sGsSi`1gN&_|yBJ-+lZ|{>m>}
z$=%1Kjjh~$td-fuh4riYJWp-yY)F}Mf;|iNOytb6IrC)C@J|n!cmMgH@BjAaKm5nL
z|9bbw-{1Z1*FS#FoJ!<;{%@9ndZi&*sjuW5?sJfVNrSja7xY7eu5v7RIlPCo?mWPcoM5*0A>&Zd6
z^49%Doo!oLD~Hb0T|1jncANmv0ziWdwINH_oECios2b3eMitezb8oUYJ{j?)mHatH
zQ`^*e9W_tZ;7ReMT|6m)vhD!O9w_%bDLYPJDwyhGM3wsBM3o+xiYLXBo;)eAu{>;y
z;YqbgAzPbr=}9ljY}>|KIdq=x+S!w`;{<>f0P1ZfY!(fh4M0;I(33uF2=Yy-T)ej-
zYk8hj2U7;nDVy5H&g-ao!Uj=_DD5IjNtAU5SoTD@Cra6If>YsCxln^E^}&iOJvbFn
ziYPsaQeb0w*cd~Url4gH+htw$)T=hzwzAevou|EaR;A21fuIF~%H?L_X3@BLBB&ct
zl^UbvnNs6}(K#v=bE-U3dP=9Zx$`<|p0GicqDs4{QW9m|1(rQg?x|9SoZwVARniMv
zDgY?9^x#xfDXR3ON`Z~#VPgzcs;!f|xn3UetGLpOI@`9hJ}RfqQ(rr~QU;x1(1Jm&
z&E(CZdGlmYJ)$f1I?J_t**n$pMX8oI%bhAC=p0XNgXcJFp18r5VoST&QZi-TEtWk~
z?%7f%ogh_6)kTLe^+Ag;JxCQ>iY-0aQeb0w*cii>#^9~DGC)^0)yGAjZQEIEr_R$}
z`^r+!suKuWAgHnO#0I`Jt>{ZP1dVA#St@rKbd-JjWVQUXF1ws#89}FXYMVQ+qvi=4
zR4J;oiz+2i)?H)S6XljF^^7{fsc@>37uvCfMTUl$T&eL8yt5Rm2K+pm~wG517OV?Z)eIlqIP?gHwe6q?L8Syhg2dASi
z6X+C9ZDZ$k)I3>(DaDj_F{K2`x_c~ppxiU1%sPRoV5%Sovef(RgE}x3Q;I1)nNnb5
zdDs}klm??@g)i#bsz$xcwr#AH20c%A?d(aJbpk*O097^g*4<;-6Xl*LW!4E!g;QPBxKb~V|M`4y
zDxwrodJ?6;#`3T+hA2%Yo9uJ0-M+tQvu!JDtwYb#UOTH&W}QILuMsq5n~9r6<7Pw9
zkVjOdCIn^E_VA0!PJ3llWm)i)PHl7Nb<{jzgDORp_E4oH%EEgrd!pP^rOZ0Psc@=`
z@{tm@G>R=fI2BcjDt)U;fem0|3{|R=tU8&HVs+N3UT52O)<>m8&r@GJyHaMIV9bHM{15{C^sM3=v1vZw4jWJZIOb=^WA6B(Zje4DJ
zTUjd|dYJx|J}6POC7x)4#N-a8Sc2d3gl@uVkD3T!M78)JA<
zyQaJ
zRSIk@4;y2sQlph}HhNW>(u+FVwz5_Zou|8YHl=Jj0iXqdx@|fKSsGWArB48L1DeuD
zR6#D?TkDhc##xh0<(tw|HnokN*HQC?4WblL+C`L-DC>@~?1^$ul(Ojrr^2Z&cwDLX
zMqKH^sfbcU=}D9V8_UDS7^2k6-N$5XWqSCk%(iW;wNvLQuboXPi%t+|L7>(q(q_@L
zc@n4|(3HmHlu1~A7R{NBJzW}5k3O8dNvgexDQTOO@;X-@$9@}2Z8PV~N%KSvh7?2E#gLLG>u#^?
zd2-8;S|*&JR4CO2izM}4i6lKJ6+?<4JsDD9V|myZ!;ot2l9GFm@{qFp4Sjm|;n(+{
zet!4yw;zA{;g_p2+qSaSPMxQ|b~dG~IKiL=gNlhFZx+p)CxiMCO=*-7N2Qes)`}*z
z%Gk<}>CW-gHh7M+=7}3@DYmqWEhSUdU0~TW<(@5N$O%$~R9!F#Q}40}(}Ps8rP$Jw
zEd@4~hmA38sXWVOv#idUnk~I3v~5f4qjKsz^|iAtWy%Q#Ef`eU?bAJk>59Vi$)ILH
zTWXz($!G25k9%dcPq`vX&*{`QcaF2>NgHG-vb2jVB~sSiVA&((o-Ad`2~Y)8UC{Va
zx%VW#^Z-?4DYEn=OM#8$VPgQkI;+R4~;Ag(~%u;VvDRiYLXBzV)QQ2Cy-PC)Li$T!Hmj!1lR(3WqSPg8vWfEgOORi&S|?q()o>1{wz+ehHBa2&N^zxK
zTq&8d?h?zMDfe6{gHDhtr0RS|m-=nfDjuYYE5((bTq&@zJZuc&N`p7r$*;}IN2QO8
zLff{q)=r(LzV@}HhG8cdv|!L^BYCrE-aHx94rohd?_L-=J8_i8$-h^BTBlFx)HZjH
zv*t+~WGS+=i!3El)*WTpBjuJXHEcTps(`BV318~v{XZ8DP(_v^OHZ;C*jOGm29c%F
z_$ZfCl`cOlz38)TJ8R|8dCF^NRm!px1X>Viu#vP`G;N*)ss>c057DDR?%g}8p7*jOGm#&D(1%a=VNsoEXZ9QwxugQg;U$uc^x%R)?i98rCm%ZfwJx%%N{8AOewQYU@Dj@KP5(%
z%1_HhmL8ajDaDkYOewIjJZy|%O0_cDA$!K;3|%)CaMn@=BF9etR84SylU}^ps6)W9P`hdC~?|iYo1*N{N(pCt3DLxu;55
zb^=rZ)$Qszw$vH1r3a{@N>QaJRSIk@4;zE1(wJgWS?64S({s^h+jiELA%bqFsTq(;=kSe4qH_y?f-pS9*%|WWTQe5fDl>!^f!^Rk{G^gmak`>~zsb2Nj
zww<+h>OAeWuPoIJJAt4Df{KqKZWfK34MBApP?kCw8`JVumoY{KCAL&T73P%B~X(
zS}>^mG=#iaG;f{^YDToBCTh8NZ>;<)r6ezA8&yV-KE+ep;5p8kCvLE%*wQYxluTK7
zlV#78d$yEmCrA}i-PR-#rZ#z-4^qXJVoOi96xdiEHU_b!IvSIc$(1XRkBdUvwzSqx
zou|Hbwxx_a!JuC=Xvj8_H;d-Y#-J(>XiJldA;}GV>y&&~D$`rS%kszk`4YZ9rBmD7
zInJ6VZIGqN(jKyuNLhH5Wsj75vXpTrKowARUg1k+c-Gr=fGV;SS^8F%0vo`_7_u}g
zXLE9uqI%J0+cwt9q4QMN&Z3lQC-}4A&t%D(RkP;FpJ6~z`iLeZAG1`2XnpYB*M3&_
zHk#U|&X<$si5e^^mb8l{B~R9UVsOusdzO@0Cnyz4buJ@Hy$vEt4@$+7Vo6Vy6xdiE
zHpZ}|+Itz;P+6PugPyB0+qSaSPMxQ|_BEx7T_+f{V9;m-d9!HVJQ>suXiB|{(VGyG
z(#nMBf-VE-98PUx=QwMgu)&q$O1rpH5@p>d2KPj{RdvXdh11(9-N9R
z#g(31DX_6TYz*Q`jeOQ)m8~?@dYf(ASj#i#X|A0$DXUK4XMvx=2BK!ssCnY28c>tQ
zwFtrVx*HQCi4VDy3+QpI*DC=&q?16I6k}~N8rh=)?MMUYg
zX=VL^saR4h>B*7;8_UDSAeJ;4t(^44RDRHN(PrCL*2;IVaV=@H
zXxa#Yx*SIorCwW~O>j!C-$zNG%1`O^DVo}*&T-Z}VS_2fly)(tB+9xwEPJBdGo{Qq
z!KrYna{*cEWzy5SgHti3n9`Fe1vZw4jX_LlwDKyblP`NJrnFjT+g8?CoHfkW!({$JyY(PQns8RRY+CH2U+T^5m|bWDy9@udNQTJ#`3T+hADM6%6C1gHs*BI
zXWMqx+Ntxj*Uqw(IVTXbKu~J~akFULJP}k6C`;viE^mxbVU#yYolQ1Y<^)gS)HZfr
zN6nKpm{Lq>7gI{0tUJWA2g*HD%AOOL3Z}a4AIMTIvh=`IOev=HWJ-aJwF(
z(R;0QD8DJaXtRwgYgwV^DX(2csmiQ5L7)YJ8XHKPMbl;@(3A!grCKSuJYb?+z1Q*!
zG^tb(rRQvF8#~8Y^P~-`6jjp0q)gqDs4{QX*yDF_t}2?x|8Xod8up)j5ML
z^(K3(4p2pvqDoJy6xdiEHpWn;PDN|v%cEE})r&gYwz5_Zou|8YHl<8D0iXqdDjNu!
zMZ@L^pk_c*DvzvWxqdINROt|sl^0-q?N-CvY-$`kUyhn5Y!Ic0(k`NuL|J!>WlxlQ
zqLfJ|I2BHHPUA|wN#aTmPDPX=N>8E`*jOGm#t@}CY9F0^(^I+rcvWZHcGlXd^VHYQ
zu9R6P81!of_1OmUX3@Oa7&PVqU8#|eO05f78>6jvL4`^Joztmp?i^>$lQzgwWN8ms
zN~A3O!r&e$_hc!nPJk+)>YT!tdK1N$9-xXWMV7vmrN9QTF@`L4K5HFZtbA2^QD@s$
z*2*Srh5K+0+@HU#-rp_w`
z=gAs8DW0^8CnZqU9b?%8<(?;H(+NxkQ=QGIQg4E&(gRcRqQ!1*~XQ%JanG&+En`|pYor|
z{5fY++t@kInkQ{grKr*_s+34scZ_9^lv}DavFQY;0;#TX=23LwJ?cz$wly%ow_Ds3wN*Q&6R3TMoBf8WZ
zC%W_?Ra_~q^yEr`jpbou5LfD>FdGDbi$2@7vsMnBr@VGn
zrOY}(pap@t=kIT9>58`WNuX{(Rr-+OQ>C8YJ15x{lzoF-`DvX#g;U$yInJ6VZg8cz
z(k`x)Oj&o3WzUp*u9RITNEK3*KSt4|UVa>I>_MuyQe5fDl>!^f!^Rk{R33{nN@wpX
z-<4kU*|wdv)}rTWubpKn%T6F@fuMFHvhC*~_|oNk=@UWqh_Y0ECMHZ6V${K8ZA139
zjG$9GwauN^QS*cisuWe)MU|2$>u$2_iE>YsGVKJX!l_OkBE*&&Ew=RFR8%Rd^rT9G
zjpbou3{|R=kHMIzD({s(F6wOC&RRQlp8DF?l}5&$V9Q}uGEAO
z*Ko@^;yVN1o8${$tPRle$ZT=dztowauAJngl!EM?va1T7F$%Y#7p
z(lsYXp9ty)l%-DjC=X@(sI3uEYMjdDccrIrY8yMRqvpvPOevs?3r@Ul``!FsY0sGYILcW`#d@yq>3xWm7ZKFu(3RBjNwY<%N{GgH%%y2=~bU?
z+gWR;&eL8y%Tl(TK+vxd)X8=makFULYzXS|fU?xMm_t;XDb2xa`LML&N>Aa`Hg;Y|
z&672lQcP(NQ%ayL{KnuODECY$!%ko-nCh%TmJ0fbEIlw4Q;I2lYf6C)U}FqZs$G%?
z;iOJiROv;VZChC@ht5-8JBw0wogmPHK$9hHR!y5HfrbG^Y4AyI+?yotROzI>7g1UM
zsP3FiZDZ#+Yo4@0m7+?!s8S+j-9?r?QtqizhMfRaK-F#U-eF6(ACw=UiYi5wo>VEY
zu{>-HqDrIuRL5zhU0GEx`fS_IS~+x{^4eFG24oS1!DV*Bo&T-Z}af2(xm3DEZWXigGEPJNha;1S;CrA}ibta)pz5JwH
zhl5mcrMS|QD+M-|hmA2@X|yqEqhn=M7<85@_C9lc|l5Y;)
z#8cbgInJ6VZm^};(k`}?Oj&o0WzUp*wvoS^2uu*)oA->*)xmwz2a%YM!jYlwwM|m{I~|-7S_qP;Qx0&!iKW3Z}}_L&(zY
zTC41NzAzP2iYYyrQeb0w*cio>$~Jb8zFPLwi!$4`u~rV9r@MCcqzpO%papEpV{oNjWpSkkry@!br6*AeY%C8OV~A4geF{$dQ1;ZTHruwc)=r(Ly>?cm
zj5>j!1%fK;h?_;@=82$YKvn8oGDZgPl?(E7I(e)#mjQIjrnaf`I%=M%L6jm&yNFWq
zWZgNIJx}h5QdXUyR4COMx$T21m3No&L8*vRMCnPC0vpT2#u%bhJD062Y{ZHwt(V!h
zjkP>;p6c4!lCtRpe}2uMHd{y5ESfbNf9gD-C4KlHACY=vbW}FWL#6UyY3WJN(bP6}
zj88%!ytw1+7rQx<+(aL<%`rj$)5NEK3bMj=bRpr1+ysbWeorEg6sumNn0VM?Qs
z^|>rvwQQm(=FaP=dBO%&iYo1*N=cM;mss{hxu;4Qbb?diRQWMBw$v;6sd+p&6;+BVJ*iS)
zV|myZLzU_*ux(7JeN_68f#O=7ZQEIEr_NJf`?^xcq7w{SFle;$-XpqnNnQG6P&=S2
zO)je}H}9>MEBH3Z|5$k``ATR1{+saxwbe`_o*_5*B1b`L*8e~Bn
zS-R%T=o3KIfTr{jb5im_sq@}M9gNL#3rqf-|Mc#|ukSzo{O;p#KmPQ?FQ;s38#}L~
z<_Q}_DWbHCC?!$WePD1;lzXC-O(!@NPIWrsO1*O8N)Jv&lp;z`q7>Lz9yZ1hrO7#)
zvg|sQJ@u;1wymtSQ|D=~omDBDP9SK3px#>IX3@CW5LCqxRjDyHg}i-QnnF-fJ}RyL
ztnN)ZwauMZ5Y7`es8UpE7gb84tUJfDC(1ol%BmBb3a2_PYs%PCE4K9DR8%Rd^rT9G
zjpbou3{|SN2|0MV22gRO7j?F6XMI#oou|HbcBL#k!Jq|$I%~OA$evn^%X2?i|~)LKj4ESfh@
z2Gs-FQn}4wowiZ?Xp}NJ`&cHB)^F3PZSEXr&676BQeT%mJ#%Qe^2#mI52g!^Rl0RA!x$Y!S!W707y>ZChC@9eSSb+Sim?hMfS=
z0zi#K4YG92xlsX7o6~@%)XHs!khSu`_$Z&3+E94~<`hkBQ|EQmJXwP$#glgNqy);k
zdn|jP-14NBStl?ROqCxLqe{I}qDl`;#gpPmPo5OmSROXU@T5BUoTK-y{G#Wg%(iW;
zl|$$0uAMz8vrYhL0ifDi!e-I1c><^((31u$fb2rZ^2;@(?4o>BAb-rCSDjAL)HZcq
zN6nKpcv3uR7f(u{toy*=9w_%bDXUIkDwyiDL6thWOQh6+sd!R6>B*A<8_UDSAf7aP
zV}k^DCi$Rnk-MU9I(+qSY+4xOjFb~dFp77o8Y=
z0%#c0lWL`$3_8gL{FrTUDOBEmIb~DZ*m)f_PuL(z5v5&3DT%V~2+N)*_e3dMPH-xm
z>Qu&+I@yxW@!(WMDWdcwN`Z~#VPgzYs)JIpq!B{p?&BlY>ulT3T03=~`r6l(8s?l}
z(1JmuHRR2rdGlmYJD@8Se;-AVrW}>^IhmX)ufLqrscr5YXU&r~$Wml!7gar
zN6IZ(Y8Z3^Q~^~dGs^hV?8TQJpo%O-mY!rOu(3RBj3G;n%2~cTGq!vu?xN1Nt*n(p
z=jpDUO(}y;0B8ZA!5YG5(Xe>}s2b3eh9np7Wk}FQ=Y4YC=*mZ>r)X-MICkyfWE1UwI|&
zqRh5!td&FO>8_nUDSJ);XaS(!8p3ALu-O1K#sNL)Lu>hiUT^pBgZx{R_g~7N)19KJ
zaq4_Y!FjR;y
z;YsC79_6C5Dwaj{qRqCgtd&FODX*PHDRWK`XhEP(=Gt(jYc7mF3Dk`!N_~{YdoNwV
z2b<+zURB=7K8I7=+&RvgCvI@1xY91JluTK7ie=A~d#;p4CrA}i-5zO2m&#{a*&d{d
zE5((bTq&@zJZy~NN~6*~<`8S|lwS1Nww?7+Idz`)+F6#e=>&on2r4gY5I2j)%@aZO
zfU-2W;APE8ZeGa+{G8+-TIKbZQ#iGao!3$GWDTYiQ`*Ip5-96#vFw3z&y+Ih1g3(i
z@{tm<)Mf3mIWQGdiYYyrQeb0w*ciiJZ6IBTA?L6xFPyQoqkW!)u~
zJyLF|Qq7A$!K>dKK^kLHW0#DjRsrA{V6e|HFubI9Hr?$CsoHb9}
z;7W0&U0f-dvhEtoo+3xWm7ZKFu(3RBjNwY?_4r@VG{rK~zZpap>{Ye<_#)8mL1sb7MA=u
z|LNU_U*CWF`Q69ge*Ec&U*3jO+t~SX(mYXvC&iO?@ucL*x@#UtjWv!h$PkrrdN||+nLBD2D
zoh5pZrE5-&ZVc-4h^AC$V`4NSNuxI6OOvjBReFl2w!w2m;XHAJEyb4hu%%?m!h0-x
zrrfio%sN4;kg8J(Vd~_a`jie*#g<}A-`Y}O1K1eDmYSfA(eiCi`AyH`qR+PNthH0;
zX|J7SDYH%>Xn~-~5;v>H%@aYxfU?wStAe*Nsi>SWO8&<(fllGnHg;Y|&672lQcP(V
zQ%az$yT`Hz$~{xctP_|DrrMm5r7i`Lr3a>BN-?D;QwnS>4;y2cQjsX~2evn%cKxy5
zX4_WQN`;=My!I8PidiQJv>?!EHEFYG+B^x=4k$_=$!nuxkOxbX$Wr4?uIvh)!>MiV
z9B0iFH@H$2vS-RISE|@`f>a?@n-jX!B`>=4AXQu`uJq(efsN&1V-Q#B
zL)7wFkMre6Jy(UcZE3BYI!}G=Y)e^ofLA%bqFsY$?l5kSe5Vb3~ZBv;p8js@PI&>B*J?
z8_UDS7`D`eZG_m_+O$)u_u00c^-(!>p7z>VmNM-Gf))rW({04fqH(h!sEY&2QmeHN
zUZ$P4k4ufQ$(5?~lud0@=XKOPQG+N&ly(uNh^Hk`-5a
zP%5GnQF;=kz{c{hF^DKt&dT_QkCiL<^)lPGv6g4fQ(Ze-Qf8gt&w@YYcQ;wHXx2RW
z(+y}zy?jg|h#YKm#>%*X$(8gmr(|lII>%Y_qz$4JQQAe65-ID>vFwp@Pn5Fi1gHY4
zHhWyDOGZHO098aOqVyz6fsN&1V+>KMZAe+k%W;)ajf+0pwzF0aou|BZR;7$OL7)YJ
zTB}K$MbqXO^uS`OO-l$BV
za!-{q>jbC*sx~`psY_aH=>e*!QdH?ll>!^f!^Rk@G-#hxaw?a1A1~@`+saxwbe`_o
z)0BpsS#<(H3jj6p$vCof&85)|Ky4b(l-i(mj?TLrqq1HefT`TQKSfj9)Oj5>PuAc`
z@uXcmDS@)?7Rw$eH$16eDw9rNDwt}sMU}dwM3o+xiYLXBo;)eAu{>;y;Yq!XMrl)7
z)41reZ3}DV(0QtBXHCkS6Z~25r?#4`Su|^&{OJeOq>reh5rr#;H0!+OPc4AnMpN6=
z`Et@cQG+GLl6JAAgvugYxO%33>hp8DF^l(OXngBA>`6!zhPX;w3no?_gOfd%^gLldV
z6)QigJI7Po;5p8kCvLE%*wQYxluTK7h-J@|d$yE4CrA}iwOJ!fT}&cO4^qXJVoOi9
z6xdiEHU_b!ItF89SfjRle^F@Lmexn*)OqS_XIsjq6Abz_gQ{#bd9!HVYz%7hfVR}j
zqLWXuBdE3WCW+6l{Jc(|(y49k9B0jwHpo(BX%AUSq%1tfvPa52S<0pppbDtktnj5S
zihW22s3J>|rEg^^umNn0Axn)iS}s7!HT?3W(u+RZwzF0aou|BZR;6q@L7)YJCQI6^
znl?`Y4FjrDnY@?hvA5+@owbX>+EC`tIh)$X&T-Z}X@e?7m3C33M9R8jEPJHfQ>AP=
z0jhwiO&MD%@F%wP098~es`R8vfsN&1V+>Uqne26vTMatpR57Ke
zXlk1}ucPM48ayeUw2LPtP}Ut|*#qU4CrxZRfvI4sO$k*h5iY9qz*Iabp7i8NfsN&1
zV+>EKvwTfzly|lir5A0sZDp+-I!}4+EJ|5*fA$!K-GYv^pRz~&&DkO
z7DAH$;4Ara&Zf4pbDT9#+Mr5NrCn4hk+SY0%N{BBR4KzwfGVJBQ^c0K=p@AtP(_uZ
zN>8d3*jOGm#!#gu8u_)DLgka5i#pr3vQ`eAr@MAGrK~yupap<>s|cG#!)60e9Y-{!
zDP@&mLrDVy5H&g-ao!Uj=_DD5IjNtAWRSoTD@Cra6Lf>YsCn*y%X
zMJulK;8a8@qVyz6fsN&1V-QiQf)Ck5Re7fLaZzX6cGlXd^VHYQu9QtD7_?weXBBy~
zXx=;-)D7rLqw+b$oO}pI>710UJpW=&>C`rNjguVpd5d@5}r57j?F6Wvv`KPj~HXN||&5
zKnno1RuMLfhRqW|^?;`I5uA?6_@IId8!LNN`Kt7kO>JZ6b<{jzgD6Fmb`hl{%DPi5
zd!pPEr7SwZsc@=I4p-`;EP9%QQxTT6$D8X0whK??>oR*^T0=FP^SDh=pLt%QztGU%k8PS$H5T$w=UbZVPB$652F4YCwj
z+C`QUDeLaB?2&RymPTfs098QMCW|k1QHd`-KowbvEIr9mU}JgM7(TKK2T03=~`r6r*vg-tc77QxSK$16$=FO8q&4{klCl`WGUcPva@<4=^
ze=8&C98Yb7=QwMgxWSfUOS{-oGG*OKmOWGM*;1CBAXP}!CXFz4K~OLsq>3%YmY!@W
zu(3RB3}Q>==Q>7;tFvWQeOwgUwxzXp>OA$evn^%W2?qU&K~2t9kvEIxjWDSE0vOSj
z%HuCdE)i&-qtVf2$)GZV#00&Gr?$a!oHb9}U`w&39c(F?vhq8Fd#2p8rA#|Ps*tLc
zMN5Qfh$2i6QpJ{HOW)X1U<26rrx(~yfB*MSAO7;UU;pqg+yCjmzW?;w-~aq?@BaG!
z$De-q@ylP^{>sO{|Kr1--v9jW<8Sg;e$mbvn^RVytg4TTLff{q)=r(LzIL{ytUJM=
z1%oE?X4$-XGHCdxht0eH{LlA)`|}_EC>
zR)d#UOTAV)SAJl3j;FT4bDT9#++a(wrCn?(nX>LM%bqFsY$^LrkSe5VV@8@QEv>av=c%uKZE0ZN2?i|~G@4A_ESfh@2DKyF(kwt^
zoRTk}qj65j+~s|+UCx2);wv0EJc=fk)=e+y2C7cq}-CFfqf@H6;QQuy#`+z
zy!g@sRFS2~(vvI&HkOBtQDkXKS_f18w6xx5+jiDRdFVXlwX-T^;0XdP2s9YE>5MH+
zE85a0fvO=@smdWnd9~Ceqjc~gRqi(E_%@u{=FV}}JaL07#g%q(rDV#w+bnyg+;gQ&
zJVC0Es*Mp{Dqm{3;15#8mEuZIt`yi<9yZ2srSiJ6%sbi2M?H^Rud{6{Ywgr|%4=s=
z%D@u@S`er=nY39nZ8ic;aX?qWNGnuqmG;N*)>IT%L-Y2cB_c?gwWZ@~wM+Nf7{HJ#xetrMx
z^^*1}o7%?Ck%RN34XPAX+C`NTDeEq>?2&R$l``xEr~;}sdTgl+Mr`Q;s;E*_=}DCW
z8_UDSAgWY8|99e2QYx$JMW1ckSu2OmQ(il(Qf8eX(1Jj%ktK0#>5^lkPXg5gs?tX^
z)88(b-_w2LbxQ`X&M*)!#yD`nORQiW7)bm&sI
zO{JKFRB@%a(vvF%HkOBtFSoTP{r%IW00#pH2D_3iO0Vi{+s;}$b)Nd#*_ATu1cQFfpedV7-Yl9o8-s>CpevQ>ksG|BrK$kH{JMxOv01~jFQ=v1^R
znB={A1<6Xl*LWzz{xg;Q;iQA=ED5LbF|
zDxwrodJ?6;#`3T+iYU$2`&j*`^rFqSt*o^UJx_b>t4bZ4P9SK3pwT4aX3@BLBB&iv
zm1^UZ*CFO8Ph~rK0w(72tWQxA|e`b*y~T^SG$9Z98l2)OqS_XIILs6AW4~
zsC>Fd-Yl9oPX<*3x>CP=>TF#KA$nuvc?26vReDaRwz+ehHBZ_gOOd5rWGRud?jFk?
zDfeV4vrd32peo%PU+TOQUwVKlvJ_c*lBK}L@~|<6EKN4~;IgVb6?ai*+g8@fq4RXt
z&Zd-ECjhhnP;a(-&dAagW$6Z>J`QL~A5jJG{pLzlNJ)F8E3ajrvZ-zCypEbDY!Ic0
z(k`NuL|J!_WlxlQqLf)DI2BH{L1ry+rB+<&!KsK+MCnPC0vpT2#u%bhYioUoF@^G*
zo~t_BwzJkwou|HbcBRZZ!Jq|$I(ZNhUApGj=#xR+fUY#yBp;RP>~pp`DU(bkfzIjF
zHg}G*=1Ci5DYCSSEG1IbePeKslzXz2StmdhP_;oWb>d5n^Y#E$WGS-rBujyfsfLeJA5?Q+D(&!UF^?;_-1i4g_qK@))
zsZ0tcZ7cb6il(-y^EzsttihAwNxOJb0%hGfmOW7Jc~VxLz*I2R28Sw@xBr|s2d3gl
z@uVkD3T!M78)JAxo?^Mnni6jR#8l#(dxuCeTia?6xjMxEePIMoJ=
zEOlOqEIl|CQ;I1)nNnb5dDs}kl-ewVPTr`>-A8%iuF_`PHrDdYd75iyP0F4V_*vkm
zmTS$J(lu8`pZMtq)TCPHV6`>cC*`e5Dd^IZo`R`u=)8`aCu^{zSkf+*lt5YcX~8{E
z?padSoWN8t)ykC`M5)UnN)JrMl440umK4}n9yZ3Xq%K((RI1Gm*Sl=n!dfZN^HkT)
znv^jo__N?odHs>BSu|^&{AmW%q(QkXi}yZ<9Guhzc^9J8q;Hd{ZRmVCX`ZA(k|IgF
zNK)ct-6sY2IJqZDS#knWfm9nbp47SITs|NbNs1&rNm5{AdDs|3l4>pY9(}GpPFkd#;orCrA}iwLYUuoeQE%4^qXI;!01h
z6xdiEHpXzJ!TS`mv#PxLc-3dycGlXd^R(B#vedBT1cDX_8cif_7LA)Hg4z*fsr-CS
zz9h{?YnQc`AtjZ|2s))x+uV5_HBZ=}N>Qa4~
z8aGb_RRgNhU~*L1+ibE(Qm3-1ZVA2#r?#>4%E5WE22+YD?P5v^lyzrV_CUF3N?CIP
zQ^8d0BeK+O|L_NjSFPIV+;{z*Iabp7i8NfsN&1V+>EKgYibm-N(w+$BQ=Gwz5_Z
zou|BZ7NsmYL7)YJIul8oMbqXnzGacr?j&=MJ4~QHt2MUrnaeb#Na$(gDJ(7
zb}^+S%DP)Dd!pPkrA#`(sc@?G9$D&~5m|b0Dy9@udNQTJ#`3T+hAEXv4f*3bE}?=Yp#X)&b-r=m&Gq$f=ZY%C8O
zV`x%il<_gtdg@i1ZChDur_R$}`>ImSniB|GAgJ8ECTzS;N;qUwAxBt
zkW2W%R6Z#^g;U$uc^x%R)?i98rCm%ZfwJxj%N{7VOsQtf2}}i3t+&Wh=ak6O15+`j
zn9`Fe1vZw4jWJBAlalJa*Of7+i!$4`u~rV9r@MCcq)a&hpap*jOGm#_*&%$QPx-I9<8=c+qCtR@Ta)^OV=lqLeKs2(%zjWg=;_
zXxcmp)C?#}UC2saCe<=(uT&IcD$i8OAM>ByefahLr=Q<_{O!k|e)#2_O>JZ6IBTA?
zL6xFPyQoqkW!)K;JyPzeQr4URRX|m)(qK!S752*qsG>?yr6*MiY%C8OW2jP_g7&hi
z5zD4}QD@s$*2I->b<{jrgD1t4_VAQpTLXR4~=Lj4E|Dh$=lW6;FyM
zJ$X`KV|myZ!;@;KjWJm(Q+`l-(PrCL*2WgU4WD?O0z0W=|!DwTv^KwJx_P-Y)V;k0zeA@4JHsa
zi-ye;K-GYz)CcEtl)Gn9cJFfvA%-%4PSMmhbzVo!lQno!JZTqCN}#Me!?Fj;Jx|J-
z6POC7S|?XIQKe2)>4B+uQatI&lL8yd!^Rk%)Ee=mR$hrKd+J4*ZQEEYhtAVoJ9|>b
zoB+@QK)nfs&7xr=0IG8w(33t~3Q5VY)fBy#8}=#5LsjzU{CUmklud18=XKOPVS^||
zly(uNB+9xYEPJBd6Qyi9!KrYnbpcoEtaa8MoQfz#l%7N>u(3RB3?fRUscIcfW%d55
z&9<$qwNvM5ubovXYfd0&fuPO=;%3pfc_OGAP?d({l$U?W(3nf|Hbl%7QTjHU+NO^E
za@0IggD6Fmb`hoI$+{~nd!F1ArHna2sZgqQ9#`tD5m$OpDxwrodJ?6;#`3T+hA7qY
zElms(Tdpjs^)lPGv6g4fQ(Ze-QnsAn&w@X#31rQpS@YyiJ)k9xUYTsI4l3BFgVVL|
zNzcjDHg%4(=1ChwDWbHCC?!(XonhG{<(?>I%?VHiRIOY($CX;S&uR`(MU)~+Pofmq
zSROV85v4|H=VK0)Pf9QPY}?LSIdq=#+EtaN%$gGfS`er)fwWmPZ8ic;X+Tx#WXvfT
zr{$NXl}#rXD`SHC6isbY=QwMgu)&mKO1qd+5@p?|1@}a`V@gwI%n43~Q{~b*vea4S
ztUfpuQ;I1)nNnb5dDs}kltv?;^n`3GS0C$bwryiA&zz^ZcGjd!If0)Aergklnnk1L
ziJyKzP5KB{DVqfe<=Gb_vrh7hw7k%8N~X4{^EzstutAfeNxNuL5@p>HmOWAKX;QYF
z;8Zx(I)f>7CW|ROI2BEbCOv6VU}JgM7(w$f=7}0aDWbHCC?!wU9bwt?
zXjF<#9M!
z9+tLRdeVzJ+qSY+4xOjFb~dGqIRT&rfF?uOtQs~?01YFW(hz+1K4^Z7M&g
zJ7rVb*m)f_PuL(z5v5&3DT%V~2+N)*_e3dMPH-xmYHh}q%Fe8~(t}eGrHIm#CWJ4Jb-ewow+HboTO1X$aBj@=C)wn%bt$an?LxgDJ(7b}^+S%DOu&d!pPkrOY|O
zsc@>b5n1Yt{K(uNoQf&Ml%7l}u(3RB3}Q-U658evYo@ebXWLfR^2~XvYiCc&mJ|G0
z@TWJPtXVW`HvSB8Ku`K`N&dz{jI!h8U2Aa`Hg;Y|&672l
zQcP(VQ%az$yTq~w$~{xcpc9x1rdlh|fh;v5OAk!NlwwLxrWDv%9yZ1>rCM1f+t}V#
zKI*w>vu!JD<YKL6k|Y
z(v+UVscr5YXU!8gxKdnc7gtKAtUJcCXUaWS%BB;f3aM^CBSx1R?Tk4{6<3NYJ-Jd~
zV|myZ#Fct2qSVS&$g-+l7239?wRY+}^|h}pjf^_Mpap{(`5lfhU2|!4V^E(4w59TP
zZZtu@DNV)(Z(=l+pVsM9I8d3*jOGm#!#hN$=y_yA$!K+S-vG}@F^j#?gokvk1O*PehmM^oF>InJ6VY%rym(k`Zy
zL|ONZ!97v#nNoJ0;8Zx(T8%7qI*BYjI2BWhDLt7|U}JgM7{io0zkO9|RW3zoz0J05
ztmT>WG}q3Wluak_^K1OvUV|iR7LA$>KV2SBliDOV?sd*V$tR`qpS-JFzCQ(1+t7I(
zHBZ)HNwK6oEGdDq@auwmpxm>hY&wCdV5&53M5)tJMCpO4SW+zMTT2RT02^aiQlqR%
z!N~3VQj=b^*|wFna_BtewX-N?(+L7C2s9beX4SNL5@;Aul$tE}?RD_Uxu8<;I#fjI
zIh)$X&T-Z}X@e?7m3C33M9R8v3+|C}Pn9z11gHY4)?{p{(?M+M0jj7{ROv~T0vpT2
z#u%#9C99L%ysvyvdQoQ^SJtva&(mG|n$p0Y698HOXf%$nSu|{(0BQ#`rO_yPpeu!#
z4E(WG58DS5K)3d^1+w=`*B%n3?`Qso0BOsSK*L^>Rl
ziY7&qo-`@2u{>-HqDf6oF*;S*(x`XYwuQAkbDrkfS(38k1b!Cy8H^)p7LA%GeyRZ_
zsf|X~19J*l2lf2-P+n(nZ-c3A=)8`aCu^{zSkf+*lt5W`fMpMqdzO?PComOEwI(7;
zopvHh4@||9Vo6Vy6xdiEHpZ}|g4#;1J$hHWcVBO_Z7XY~LeEoPJBw0woFLGGK)rFK
z&7x_u5vYv=ic%d+h%w502+nGqeaw~H4CicW8#~8Y^P~-`6jjW*TY7*hsuWdvQl-Gg@~|<6D$OCw*h{RL(u+FVy0R7>dYWm|777d#xfVu%q={9mN7a4Mtt^1H<_0&~9DLq9~+thg-HBZ*yN%5pzJSl;)
z?gq;qDEB-mQ%+zim}-qjl{#&lwg;x-N%5p7PYP@-4;y26Qk{G-KDeyQyN?%bwrypt
z96C>V?JP=}a)Lk$0=34GHjAdslR))=qV(a6{6QZ>2sVT$50%P0rDXt}!>MiV9B0iF
zH@H$TKJ}T03=~^4iyxdd8d}(1JkC24v_`yP_`L2-KtjU1_#Xen=;bY~`UU`L@DW
zzA1ehO>INx%SrPD4VDy3+QpKRChKmn>}hh#l6t0`U{n~@%6m|VQmsUk9*l}5#gd*Z
zDX_6TYz$&aQ%c(xJ(Uki>rJ+8UoFp^r@3}kq)a)1p9Ox(%@(3&(WrUiryo#}`WTdx
zTlc{vW228EDa)@(C6|t1Y8yJQqvpvPEGd?>izOvc)_qlQ50rbBlqn}L6->3pAWEIe
zPOAe`v7}hilO+WR2J2XHruwcRt}w~yml6)OgTZI1%WDg0ti>S
z=Dg^WK+S-n^x=&*GJEescFH7|baZ6^ox`bZ?i^>$6F0b0Txl0qN~Wwk!m?+|Jy*(>
z6Ql~MTBFgWP9@Q$2dUypaiu3$3T!M78)LZAm}FJMxa`ZOdevv!cGlXd^R(B_vXnI^
z5cF#V-F|x$H;cy2hM+o+C`+~MYY7qsCEt{0>$NkLM@moW)HZisN6iy9s8UpE4^>K{
zEWE?AC(1ol%A6CN3a1MGU`w5fVoMKBMU|pT->Onz1K1ctm4+OX(lORl=|!7uTUj5K
zQ|D=~omDA&P9SK3pmM_*Te|4R=o3N1h^jQi=%QS|4_Z9F{3mZK@4uYVscr7Oj+!TI
zP^GBSE~=D7S$BqIPn3JAlr<+f6;8D}V@sV1VoMKBMU|pTPpTBySROXUP^DU)jFW%K
zM^}~4N-yec+s^u^oH|c^?dwV%drmND!JyGt@@CPzc`~RS(v@oYp1`H-jmcU2Ec*s!
z1fAolZSWjt%@a4+Qfz4#TS}&^JIAtT$}L;!SapI_Ayumr!qh1*!t@|jY$>+%WJ`gK
z-762-7vUMxP9-2DGJK
zsc4lC`rN^!9k_GYRKO8)PZ6w2LeyQr2B%*(2qiEM?pYPz6-2
zj`&h1_^12o7g=}DFX8_UDSAhJ}3oxJx^rs(8v=+nCozrO$U^Sh6~{rJ-lzg+a$
zww<+d=se}MvnplW2?8w$)Ei6MESfeOfvPy5Dt*WWv=EigIe4Fc7+P0;&cIrI!wX-c{-3bOQ7*uw1$eTs;=EW1ov
zrKfain>+e-);wv0EJc=fk)=e+y1OiUq}-FG%sT<9fU4EX<)@SVjs>o7g=}DFX
z8_UDSAhI;;;BpM1GOlsaXWMqx%Axa=*UqYxbtee4AW&;8X|rhBJPA|}s7fCu%1`K`
ziPi5pxguC$9QB~#WNX4y04o-1YF2~vert#;^Ar?gY%
zAXQu`uJq(efsN&1V+>as8rI^w#rj$Tgca&uhlv}3M
zvh4(>f~i(pWT}(q>Xkk)6;p~SJ(*HqV|myZ!<2fJw>RNbEWhcwD6?%FYvs^+x@%`o
z%CHjvS^%hAZzgOO4Vx!``T;#@OevTssS{i@*(YZ!-;|!Bscq`Kj+!TH@T7RsE}oP?
zS$B_R50rbJlvyV*6->4A=P0T)XHlgGrs7HQq$f`bY%C8OV|Y?6tSv7YDp$FEf6->!
zR@Ta)^OV=lqLf)D2(%zjB|L^JU2|&mNuXvxQ7U6jK4z7INPMND@zu-sr)+8)J4X)A
zlQyVQRB0DgN~Em&yx<-w_f#pXPJk+)YPH6e%6s)T=L1wxrKr-ADg`!{hmA2*sSEN-
zoa|XtMx8F|Y}?9OIdq=x+S!yc=>&j&4WLGT5JK228a5k%raYi2m6xeh)HykMtu#dE
zyietohEp`PO`X?K^JER46i?d2lM*NkFR|=_a?g`8=me&Msa7jgsmLz9yZ1hrQT%gW0XIvgAcZ
z%I@&vqRzJMthH0;sjr<~DT_`pXu+Vt81iP(ym>OH8qk&M=;W(Xv8BNzG5H~-T>iA~
zoK9_X=QwMgv_Y04OS{NYB4yn%mOWDL$x=3*098QM%3T_KX?8aI15}Zv$kLN61vZw4
zjX`9o_Ca2YlYNa!Rjv2g#+|i1be{6sS(UQs1c4R=>cvE1OV`{P-3TxW!?K6TJwwWx6Nn0;TIG?X
zR<>rdJrET`iXlB2Qeb0w*cii*$}Dn-KBn5u`->*qxUiP>d7kpxS&}m61c4R=YKvagVd-tboY8yNHb=Ev-gDORpc2T87
z%DO`=d!*b_rJ6k_KowB6%3@1xR$@yJP(_uZN>8d3*jOGmMp327B)OVfX{xqfXWLfR
z%Axag*UqMtF(&}D08lMagDhQhV)O~1en3;|qY@z0Nr}nTxjH7Onwe=1CeX
zDVDU0B_&SQU0~VcpoTs;THl%Dg0iOkY%Kw6(Su|*#@M#7#q>q%9(_Yraq$GPuAc_aim=wDS@)?^vWJ6_Z%q;PGBmSYL!Nl+9dZ_^MR>2QXJ{Ykpdgb
z!^R+v)S0N23Cfit8W(N0ZDp+-I!}4+EJ_)1f9Ef0jhwiRSH{bQxscz
zfGVmKRr*$y0vo`_D5_MhJz8sHrK(=^*|wdva_BtewX-T^$_WB32s9beX4SNL5@;Av
zmHI4OPSJRy#F@%}a=NBU-)2+W*wNeH#xn=!NgGrts4#tR+iYqZ
zJFlbW2^&NyqO^-BB~jL0VA&JpmMB#WIl-xLs>uRBxKb~!^x#xPDWdcwN`Z~#VPgKS&i@
ziY-0aQeb0w*cii>I_Zeo24~B!dL9>jwryvvojOl@?JP@~bpk;P1ocJ}H;cy2hM+Ev
zC`-54@h->cY}P3_l`A(JPU+M(cV0)$6E>()RB0DgN}{Yg$g(HOJypuC6PyaC%EcON
zsZH`TYIkrdsuWdvQl-Gg@~|<6DwS<)mE$b)(kFwu5nZWVy>eQ~Ka-J9OywTFtxOA^=C9m$smwm61<+l&L%y_qgb@Z98l2
z)Op%#XIaX&69`%$s5LV2jW1Oz{(SCh@MUwlrz6r3a^?N>QaJRSIk@4;y2sQXP`=
z+l!^OOZXRcwryv9R8F0zzIJt`F|+OjgBA>Gj3#du&6|xuZ5q*)MwN{V!T1e~a&RtG
zBIq1XZG-1HYo55lmSRi0*itfO-C>qJQ|{Q(nAvxNR3TLvuSA$ynU+raAXRKBw)A96
zfsN&1V-Q>Fb&SD9Q~9jtaZza7me$&-^VHYQwv>G*7_?weZRCbA!gR^i(IvyN6I}}%DfYx3aFZF@TE3p
z@udf-B1@5_Cs_(?EDsxF$WpnXYvsk#>NWg}KHIjlRt}w~ymnTltUE!V1%WD~Nt;E}
z=1HJtKvnv1DcGPxNJ%Ltzh>p>?DEma?@d4mXDYGV>zdXOrv6jyq3rNGAWurY=!bvZ|!Yp=y!)!DX{wRY+}<+ZabW!(t^
z{hC1Kva|do1YNqlin-inhK)c?9?+HQtew-@>10zfDrR39cY2#mZBysVN%JHPniNgi
zLz5CG3%@bA$H_fS%CHlV3Z$AWV1_A;VoDE4MU$dQ-)d4|1K1ctlcp&54!v`+{GjKe
z%C;@6l|$$0uAMC@vrYhL0iYtG5T%Q5jXnW13}{IoF~sPCNK&tDu-eJ%5M=lsDdEE+dY1hoUI(jf0+t1RQE
zDT*oe@^xt`O3&HUHg%4(=E)jVDXR2;Ta^+hYrieH2g)r|nwWHgQlV72NP{blUR>!x
zsfbcU=@(H7YJeK%pWj_S{r%rRefZ1Ye*MG0Y_<5W??3(a_doyJyT88w@uwet{PNef
zzw+_#|M>8y_dmb;_?!HdUqqwGYs&IOY2}O3dY5h6SIaZ!>8+g|DPvC1C-ez@MrWgc
zqNf^Akhe||oJwsI=QwMgq`{D4NHL_wd*g%g(fDM1HbI%7O)w@{
z6PyX&gkVB6A(@a(R3>U8%caqp=uGq`1|wT?F`1Z65(L&J8I!C@&LnSA5GXdp$=f)&
zr$-ra0#E@|{zQ5EC(6cL??3&&?|%QsQ&jx&`|nB{>wJhwmQ(chUf6cu$jk4XsFTh{
z-Vl=q?X7V}{&<%^&gCsFtE^l|5|VV*%MaZ}XxQwO)6O{SobyiBLS1s%E4e#sy!Fm|
zAAI!5=b+@Jd>}ByY}7Uy
z`3)<-)!KE|FY@z$EJo$y9r?hl@&V69hi%(aD~HZgSUamx#+x8d$P@C6Zu6o#4(LT?
z*Ipu0g3#w}>fQu;P@Wyt-j=2GqPezhr4SZ=E)Z{C>j(EiU$2((4c#W+;gBzF#)6i
zDL}gYZ2k$P2v7tl0(5Bs+Nhm=oBecZ``BD=36=Jb8C323ej
zn$KX-18PsX2O)P^m2i*Tj*urrb*NoEc$-9RW9Ail^TZ3{6Y+`oM0~aupL>Sfv!3iK
z0i(buFsh$0itqGu2redGAzSIh9vUtu{^@FE3cjGu84IwKnn*fRgV3h@zbEa
zV0O~-##B(-A4b9GuzhHtgS@aRFQG;uU|<56_}_pD`D5TuzrXwV@ykfjf4uvzcYpl-
z-QRxwjMRD>$Cm(dK_(5l*9{P=bGp1;jvb9&#u6k+P?pZr^
zo)+5KaI$0sT|$>j(&aN&%z%1xTh7cWIoV%M#);!p!_&yIU$lc^%u6ZTx3J<*zIO|<56TC-94{6?qgWI8^38!PW|*2`$y!dadnDYPoe{#+Py3B{%$w|>?>thV^wPu!8_KM9s1yI{?ky}1{*v%6v*i39D
zHWQoK!e)L>j(aAP!6B#<>RgyQv6xs)EaqAkvrXmvHiH>V%1-B8R?Djb+O}lYPMxQ2
zc6OMo3PF?5+xTf4EMHEVCrogcxJ%q6?((1R
zEo{%UA|Re;`drb4k)fSE4J?m8dFo|n
zTgj{s3<*On!;o8&r~y@F*ftgA+e#;MKEZf-^Rjj|{1iQHLq)&NnkPyym6%FQC8qM9
zXevMF#XU{QY7dABVy=gnI7%EPj&cb{*``*0o1hHJ>KL^*rJuZLnQfb7<<|4On=m;5##>9SEKM{P?Vc~LFf_Q%Sh^EAoMK9Wft
zpb=*Mkl9j{!?V+=bX4_
zB3aA|fxo#g|2Ym7=ch=Ej<5ADx+(>6+uv*t+<93&1B2Z@8c?;tBlU4aG<0)v`M#$@^d2$24h+o7n;upW!FMbY(dvcKt8R!vu
z{NCvivxr&5EPjV(u}wexHmw+a$l0V=*%p0V)Wx=)uy*P^^{}&6WG4nN1Ps4NFqF&)
zc|fNaObT8eWz=#n*(zsksyxeh3YoU?@j7ar%s?rk6j6#O#cxxJpM&9^Ph<%OYy=y>
zYivX&A`_8`-;ht5+4`?^ENSPLi!3VvTGxMhVMP#iu&
zlB>)n+hn8nMqW)+l@HnT+jMCg9IvD1i3`jj<`8p;IsBiR!_S#;OB*`oSYSn1@jGQj
zoFUE-XZZa%!!{-G+k~O)ahho4nXOP7!>dx*w)xdgou>qLc7`mjz=3e!cf^5PR;Uq$
zp-Cx-%QRkvWQ5ORuFN@~)1_^09B0jw7AQlMA<7VC_&-vHpCjR(FJx5(PJ|P`M@~c*
zA`6j)--#@2)Be8E6w0M!D?KmNhEp!uVB5y`sGK@a_v>s58BT!!A;9m203|4V98wc%
zmxI<$zDW=Au%UccSpJ+z{wr^irEO*$XU&roSVAlzmJmz$-?xOH4n`KfP<2tA?>
z^e#IuN-!pGtkKHX){n&zz6q7KadDhAPe`Bx(Shhdbl`tS2Yyb1doGZz5~vU={Eny)
z5r_yx1kNo2+f=-7vw+z~Nk|ncOU93j#@Dvh)lQwK-}RM%mOT=158Rs^_aq)%8qoj-
zBiDl^FoIXcN1JjbG){TaHYr|5%@YskKlC5^5B>N5tN%V{z%BP@85{uyfx+(u1`&UV
zKg8d};;&7u`!?$rOY
zkK+1LFghfgDqo79lB8`&ypEbD8t{GiK71d(@Bi-mKL7cDPxi4E0tG^W-vtF?`Y?T%
zzFAFQn=d7r$rZ-^#{z)$7`Jx7w-mw7AZOk4X>c4!WC@?j#m$9#HS;?6QjT_m*|=
zI+@@rKM6YLNZW)s&YC9|uzT1&>>hUS%I
zzD?>G<>ana%)as&#zmWJ+}O$rIZt)#tn}FAfa;*S38`*NgfO7bd&n5Cu_0+CLmck^
zu=g&@k{rjG@T<_Ul3-S}xL>%Ol~&rw?BCF^=wF19l#!qe0O$F~<0LBUB-69|oQ(2{
z@T!lLK+dqyewqsZZt3A+Xt295AAhzSB1;dqT6$OkfgS=q1bPVc*g$WA{=PI1uWms1
zLHB9aeTec9S30+FSLwUiz{K!k?~4-p+s42R2)>C9u-Ga;PQ-N
zor|Hy;9X6X-nTH7{<;tAw&Wp83%6Q&$N+I2;yT22i0j_RbqnJ6Wp#MR0xAzGPoc^~
zP=}xnL7k4EPR5#@OX*6^HqclHGg5asaB8w{jH{~0mkkc-@K6Qh8|0f3`POJ4OqkGR
zM_+D238~gd8DcV}>;t+DZ^$yjT1pQMAf7`!hjQ;
zL~*l6aWXFKTn^`~&57;@1?GyUQAIvCTwgFXYo=56dM%V^oot}-2QTE=J{*)dtm#Z}c~z6J+yc=7>a4PtF>vDVN(PME$qx{vC;
z@w7WG*F@XnJo8CrLl?3{u$Iz;{)pZXy&-x-^tM2JU+#uC9U$c(<&;P{gl-7k5W3A9
zy2*I1bBSA^ZE&>kr`G+=(*p}8>$8}mdK}l_ybVt_K%_yWO)SzH@Vg1awu_|)Ra1%d
zPgkoAi3nlO2OFx8Wr4Mn9_&ZjhO`Z7+l!}d3&8h9ZFrIaDh?`6gNj4WhMWyK+oU<0
zjMq9BvRRX|r#TB|z1xxXlJ!?yRXtW~aLk5B7a-6e&}J2Ag!`sWnX$!4_ep)p#yXR#
zp*iP$#dLZY{=Mb1rq^#s*O0CuU7Imold)Lm
zqP1W=?d7qNd9vx!kv5)7ql
zjy4Su;QspM=Rg1W>wkRt&=8=@hKjPw$Y^9&EXX9!vBzgE(N_sG$ZqajCo%^Yn3^{x=j;B^BCHVC$f
z1sf7HBxp#`=1S0H+|;@FELJitIdI$54$PITg<^{8aZ`iSGrV2^F$OUW=PGDn!Q?Twm^DcWQK^ZfD6nZ1Si|`+VNP}lB@}0T
zAxV&Y$|e|Q#hKkOgDmLZYU#mx1Z4=y5R|=CP__ViUrL780-({L(dN@=h{zC;AtIY1
zB9rk==kl@SXc=^jTs(GUqhzfTS5=Q`8XS<}JphO=h_5-sS3`L_VLBFSiS$jxL>DO|
zGc>~$_Mb*Jv>*%kYbiY_k7x|h7^1P)iN+Qv@5{xo@E_6)(rhwmhENQl7(%h}p_q(U
zI+uvqYUpxh@s@ca^T0dF8YHHu9;-Ao5K9#+{h_*`x+YLvgz%+In1-bkb4iY-#8S%D
zc^c2%r(s)4kVX4jEj=KQJPdgl@~{`l!xkj(3Byvw;(o|7$g-Ja8Imw0VMxLTCt)%s
z>1+&^BIPIEGcnkKX_7TYOi?`!X>bgN-Tcs7&|9PRR)hCAVFq@w*#}G4BvTD0`ry1_
zL$LRRAdB_3UTf)rc!XdG!4QJILI}3tcwYjB{rga4P-N36GQ?kqzYu?ojK5^e(aG$K
z_QufbWHz|{EWnXjk~KtJRXx^daQcM}`_Nj@T0^x~Lw7f0^yLgaKWU>$bQ`jooI_x~
zci0Ud$Rho%mL7aZ^o8gP(bv;QUkix$Z$P4Bl_3
zKo;cJQhJabQ5T{vL|xAvbuIASmvdnaJ|q|<*aQ*`As0d}gj{1nE*T$mF5$ACcDvB3
z7t1^ZdEkp=%@0#lj}00eaAD6q)D_g#Aa&JHJx!Qy*<2}5sy=$_bM+<0xc{25r2$!l
zztz$M?8vu}Zz11$;(TjC@4j#gyX+yqAiqY-FC<$?wvcQMNVa4Q(79O4ncyhb@_~88
z>%b5Ro1VQ3)8l@|###m2>Y=Nkt48Rm2I+pnOzT2FR;n%$${SNcF~u{hxwa%Ai|@Bu
zdSD%)7D6qATF)A4E!f?aXca7^hw_5*8mhbyXCcl)ob~89OUC$|%d+SxdY{Sb6z04A
zkr9$LJ6u&gwr6mfg{AY*RM1op*HjJB&4f`FT`rH=))HbH&B`>3%WPNNQh+SD-)iZB
zbR=0wvXEpwWskJOf|h!$mLf1udBW_9Fx~qUigU4A6J1Q)miP_&%L2PyYw5vrWLL
SG10qM+3$Cy9Jn4?XTucL
zV{ryYR@nOv6$KUbKo!-HJWQBZW!IjstvAifnxcw1Rzthm=@UdT`-o}-`?83>mePac2&fQHA)tDu
zfNBBjzH|!9*&(kWuO2S15KSSPLNxV=XiCP>oXe$Zr9~#jGWY!s{Ee`Q*_$doc4llW
zm9cysx(T}JA-ahWJeCPFsf!IM(OxQ7XiZg(vC!)EeJJ%F^JU@uT1gLvBa%WSg-GgY
zBB=$ayYi@v)#}hy&{dDsRS2UHMj?z652Iv!%(*1W63ftpR0zywk|SRu>t493dYsJQ
zGz!bpp_-tYb@un>(L>qAgUfHs*pn=he8e|n?uPsn3ExtskVetggC!|i=Q>O)@`ywZ7Duu
zVa{}k-W6-Tp?y!D9;f4GCt*GzC@11MA|-^_q%70e2c6%;i~F!DPt3+guUWWM^HyH>ZrlCpDev^&_nmuLj>18PnaQHVzAZu
zUs+M1M8NQT+ydgM)H{RdZ7j}sXj9$`l}q!6UgU8PV1>o8$*l#;WtR$EK4
z#T!#2bDM30?y|VOmeK>(h>Z{%AvRJM8!d?3ml7n4)^@$Kb38OSU0_Ac5{Dff`QR34@{w?b9~V)OJ~o{;f0C
z`2C#d2H9mndo860s*w~SDMC`DD=Av=xGyHc5^ZQCXe0rRgp3Fo5i+9tWJEH?<6J;Q
z7Z~WD#X05{!;uk@bsJn&J;q~fMij9o8~O+O=T7>EaN3m_10vc$=LxlQHmUgHDtnK3
z1MISpeIpS)K#hb52@w(^MG4V@#$EAH#3F3SBgmtB%Ohk%$cB&&-60#2F&rm@A$mKF
zHeIB$|MYy}{fDf_V2bK78-r6JY`TW*f$X`5>}d!cXN-h=s_j_~AqPyxMWI@P+kOlQ-K{7VuTo6Rh<+$LT
zXReT44qS(6h-=(1S7
zmeK>zh=C9TAqG+r11%`rmjPiJH3Sg^(ftGw0w4rH2!Of*AQ^LUF8ztwxk&d6!hQ_s
zz-!1F3Z|$YYcV+c!Ln&c9Y~!%sYBRo>x9A2#pkwG+UI0yw%G<7n8)omXf6xXYbibO
zjN}K&50anxlb;2H`(hvLlZF<87P^xbLgs_a2bs_L%tyvloC|zx@U(i=g~Hf}Ba!T$5n22-Pjx*I|x(tx(-Yy)O3R&a8Rn=o5
z2B$mN-wbU7ZF5T7G*mV-Mmx?s8ndKaC)4xlA!p0H@xFy|S%ALP(gVv#c985K*_k%k
zS+KV+*1?KqNFPX_KIwx@2bm5solj>vGS=Z-phN2%Om@{UYj`falaO@_Oi?|yVQ`v*
z70i${kTjo{G=#?{PZ;H-8jCSDhwQEOgwxzaMtFnp2I0+w;mv}%eaQ{>Btz^#
z?3@)lh;0ztAh!8LY$Ib6&Sf^K4d+K^m_^73u0hrsFh%v4gu$5&)*wU1K*oGV#xx|h
zGX^%!(%n`{Y}(9?u_Z=kCC&!AWl?!8r3Z@<*dVY$U^81_vjA>iT7xymkU5Y!r(_PI
z8bmdSYCam($hd=ZIZY_^;B7E$PQ#l8Sw~>^MfG@tu{n*$9%Bd?2$&BGm><5+wM-b&
zTzsnZ47rKH)KF-^rY2?+*#@{}F?lVe2Z@o=Af-V{GgV5nz-?DVz8ALLOWG0DZ7O3sZW3V$Ax(2%DGr9(048jB*uGsFQnSStT4D7xB^+_&sEjq0R|^ASPTrs0>$zz#XWnc=4vEgM
zsnz;OcQ|7R3@qCXWXpo_t(G1LMht@(1~JSGG0cLqeHje4|3c9~(R^6ZAb>#tg8=4h
z0gR0JKbgMJwWOG8h};~^ffLG9_c?geZh)fXccIcuWFTszG=ef#oOd*BGGvp
z9VrO@1?_f=^aj_mKzysE2Y!*fAbCOZGCX-%Ft#st!5Uvk8AzFrNf~4=$Xt-Q9A+*u
zlK*7j;w$YP$fV-;Ygd8N9GTXSg>su>ICZKK%F$yH52A5msIEu5#1#6^d=px
zb@7z6?6KHxa9S3EZ?*IwF9H?>EC^Tz1}qD__N6P>s0;l9{qimSf@lTN3ZfMzT9Hxs
zCvz3r%xK>pHQj6ZBgHSP>ba_V^!?yu1v_)0OrT6SWkR5v^Ng`daI_#dyId^gDlXI*
z{2q$!2Bu{J_*P2~>>^e{tb$l&RIIXKYhR{hEt=yIU*Wwkt4RgbD49HwCHEhGsf$xf0qzzq{7DWzIxXb((#>CNSoZDpQ+
z-XOFr_O7M$04`z_#3+bShQuffs`h0l*lG*G0>N@DSP-BfKtX`A2~cFj{JHdm79`U2
zxT&7e?FZssR>3nx^=SFQ;R&|ZLW@9)Y_&)OTRUNLaw(zuN{=;{>JlxfrKj7thnhF=
zEQ`E1D$oPC2u~26AUt_|c(R~sUvh%Qv`{NhD+g)?u?b=m#3r}cL`KG+%S?h#k@g*0
z<|_V?s+U#nTva_9er#xBv3nLW1Ty5E3?ZmZWx~Y7$Kq+yFNbP;b
zYbiZYi?{@F3F49m$0ZA%c4Z|NyJaC#AW}Gyf}jLJ34)T}f)W|&el8`kZN-VL*@nGJ
zKaldWdYvh%N4XCUNw7T@`UCposy`ac#t9RWP^)n@M&l_a$uw5tnS^AE%d(JrtEC5N
zk&z%HK}Pb(jAVh+zJLVlVIfc;P<8?Z=?Kyhq$4dIkc9ueW5FNtGAJO@pvx>&Lw@wgo5b6E|}Rn?=$2ZtTlh6;)C!V;qatC=zBs3y~n
zkL=oxj}$X)LG^pQ-9WJ{$gZXIAS-eXKKnurtUHAQfdcv6cStypaJ*>3
zA)~_2#T$u4w++TH9~LhMqFh#ab5-?-@WJs0Hkv|QysEe$R1JB?Y$In|Ql%xPKGXh=
zYK>#wcG=*tEW)m(^uQ{z4P+b0HsrF61v~qK4XiAM_IP>ifm8#j#w(^8GUEGOq+wkQ
z^oU}PjLg0qh;doP%~jPSz6WO-*h>ms@shftVQQE#(D2SVQ=&Dt&_%$MDAm}{YHUGR
z7G7_)^nfZ74I~;!G=vh31v&fT4D23-?09Y2fh+@A#>-_HGNSurkddr6F1gCwJvk8J
zvO=3Fsz-4Tjxn%a6nf(I^hASGJ7I?5vWt!$@^Pyg!zmtk!1R~A~SIex=>J&1}7
z0~rP~42cY5fz7@E0~T9N`9-%!n
zx`-7ULZKvHOi2)y`Z8l~LF+(m38t7DX#S#D&j74V@4;6VS8uiSz$tPI7I!#p~${
zL=%W6ULu;1QPk&hiJD@e#rBceQE?=&WkobsRgat=986%@CWOOl2nWJYS7%Hmyv?}=
z+OV06$;GA;n75fWs4I)7YbiY-icA8T1Tu+xWfBWg_5~8yrU_l~V!8rp1k#A-Pa|Yx
z^tmV^WgmPnF6_VJT@F;WtaRq8>QT{yqX;a_gk*RD$IQ9PfpjgU
z2R@NJAbCLY(4Rak7}*s&6s)v_mUtO0fy@D!!!u_NGNSoh;9!%_!58{6xBoDAB%)=N
zGFMfPXdaw7U~MII!c*&nhMi`@z`?~}VhDwv12x_iXAN_gZ40)tFnX({2RxBDAaOwA
z(3vtUzXp>QTyrV+O3A
zgg$s)eL%2j^Mo10#TOI2r58I5&1IQn9hchL4baMB=&hC>=tRhXkO3h>Psp&~V_(96
z<&sbkFQ6b0FCboc&UisaB%jO{T%|h`^sWW7di_W=%L-$zsve0vI99s1CgSm
zI3b1VYAP&@zQb2p==`ma9=Jq?fD8c{!nq7#!Nk4*0s9@H7oJ)#AU!~O@NDUUj2J!_
z9oU>qq;+GFdqCnq9Ls88PLk^p#AEY=j3tiH15c+12ro^UFg(b{l;l0_YNW>^LNK1&
zgS!P-S?Ijg(gT=C5Rf1sK{$~hEU4HOA7t!lgj{%Dxq$2d*}+p~2QotVTyQ{>x4za~
z9P?89fiRX8zf4g*DtK^gfIW=R0neoa8dSy!GlNLa_12tHEi`bDim{P<=wXAcvaor>
zaXn~>%mA4IGK0@%1`8zi1qRr(2%+$#LIG(3(t_tn3uN@~xu~Gl6xxuj*u7hNloL~d2vY5G+(u0&p36K&X
zCHPcIu)twoM1WO^PzcYc5ReZbA9#{{Kt}bR3kT@cb6Zlebsr8KiC|f!%T?7QdI!e?
z*o6r4e-`rp;rmqUjM+d)KE~*iH>uKFxZZi@^411dWf5~Nr3WUF4ImppHt_LmV8Oz^
zU;xVwp$(o+8z2=xD)0=cfQ;BZ7YW#yLZSB#nVsI31MMp-aJi~_wC>XQH&je9Z%u({eCOJ?p3Tn;p^tgz*(>e0AkbAW`^gkXIAjNbs#&lm#4nrN#)
zb+nX)Rt)4~4fDFJ-9V}=Sl(*sfk?yvhyf4-d?W@~aIh-_NZ39IMesz5009630NnwA
zjJQ4N{!68u0hJy$A^=>e^`H5|8H9V1qAziKb8nW3p|S!fa!
z7pAHnAv@UPW7{A2T`j*iTnrN${8Z>!Rf<8Yx3mK#2b-Afep?`wg~(eiJ-`Tu4~Gwj
z{}qS7pkJT8$3j2Qzkd3Mw}-b^>Fs3%>^Wm!it}{iG&5WL5A>|8W@U=%5wC+?Jr?$X
z)^*Z)Lqt2HrKkJ&CdA@tX+K%}NK;Px{Br}Lvgo*$(u0XG^f2@=^hbt%0lq#rkBxkw
zedV+dD-SEL&&tcF)^k3-mP!vwmBjgYUc|~O)wD0FN3)Lg@evF6K*mT}GsyGwd(F;>mk2D=@
z)3LJ-1g?j`2?tZ1(W6I89zDij=+k=2HiY2!9({wHvY5D*(gTF>=y?C|6aFBpvL{v3U;Mt%SQ93Z@y2
zd7zKxshCW6flTs--h$^IgWuq$EFw}+@EbzK|mOA7;qSHGT;l$^|^1Xdjp^A;d5ATSnsK=w~X>U>AQ1`mcI8y
z=6T;EB`T{nxvF|}=V0fJjc#D-xtW^4FXkC-cMi@5`XnEHOocYNMbErSVK;Co3xu~?
zdf*SX8@3y^d%?Iq&yB5ZU~(l)4#N$@J+I-G5u7L8b_s80+cIBtE(ZcsR%CKj^~lY^
zZX1i)z|j+PbVI%{qt&kTpjn#@&9tV{S)X$;`>7wh!Ae;ayw%c!eX!cF+AnUk7l7;Y
z*;u{?4%fipFxfEKlbUQ9oq5t>yHqM|_O;Az-^+pSl+~78RXr+mu*1fBHL&xn?A&0_
z&Sk^())!1>O3*H5RCWy{Bhy7;6~o
z8I84!ygccut%>xdrh4XfHmwh5<)*Bp0^7mO6(}3x$f-
zWx6VpOR<*uV!c60S^TqWEj_3QOASl?vX***wk|*Iu>}phJvVQ|Ov6l1XQpLTfh>+{gqWd_Ecn6Y7?VW8(S&@y83qg=jj`>sfAo}HuQbha9^1>$S4bXcCDoc>)@5)m0!&(FM!r(l(A0?
zBt0cb!zIHd&*GA0l;cT@OgHJQqi0r`B{>I5QdTW8MfJ$W!4?_&!oa@Mux~@TG@(DH
zXQfL`-h29(9F3`od+2I|in4&$?lLb3tOw^{k718r${sIZ*5{3}4-D))A3MVs!x&Ft
zjAiuVIaf?eZhXjr8?@%-qpU>S*VH2w$68{CWnUoOIY_tRo1f4RUy7$2B`H+elJG8K}1=!TTAHyIrw4t;n(rQ3zBu2VTWa2;N{798BQ2Zc)Syqk%{MQurq;Xr|HHW
zbKCAnHp*&4uBslHIM@qgT^Bfa0?us+XJ$0Qw3v{ti_jHgS3ONlyHpw6+hU?D+TCjD
z!8ll9Sm773!V8e~`Cu&M0w>SJ$uPk%!Gle(j6OW)fNd?gx=G}{uqxjdu&b=A-{kd$fDmUlMM_zO1~y#Yj7sJnr$9)N@Wh5da6`@5i6
zpZCR9EwJ%4Yz*TI<2%y$%E-cVu9p_JxR~ft+J0@;A^PmUHILn_qz*#^_g94#sUk^!NPF5
zaJs{su8bBuXLGGDo*v_jZZG;S2hvbh|8Z6INWsB27t60eu;U1};hLS$<7VUO+Hs9!
zEvt#C#>9kr8z7X0xV4lXT!Y1h#eHszyMR}pzr}Vd(C-BF3v&x|JI36~D8Q4>wuJ1B
z4T-tHcAyAlWgk;ij{qENY%Nw;fmsJJD`A=~6S~?#;y8hkB$?GyXj1-rUTt;LoqQ158!g`RES6A#Qb$l~!?QT0l`S>XLqsL!>`cw{Y%lYLc7uSj;I@|118H!waI(+pWEb4(
zv$0q*1HeyT=M|UY`xAXow#RmUmk!>xd2hLz$VPBupzAl*6=UuTR3S>KuY++ns
zTpwy&Wn|tt*J@);(T5V43$h2YPgdJ8MfIq>!KM{kpunS#OlU_hG0>+{W2{Tr(bHWaGO)HmJXtVXOX-0ym{FL~r!k`oPW3raEKLHbK9*Er
zLt#V3ZK#ZdJLf@j@jjL4{N95eC^}i$#uU{f;0Ak8EJXrc%G2c!-^ZF~G@!Q7v~J3@
z$u|XCOCng?gW3k~WT9*=r3b*^K;b~2!+|a+)n`Ak_XvFYKt6@{g!h#7o-)$yobhyq
z-jm7RXXZ1(NT#a9_cpNdt$c{xKfxa8+wH)jVFEW3^vp9-Bjrb8M-UVOyX|w
zP8P{-we;W@tS7AJ6Ijm$r22d())#?GAH}6GoiLq(rc*|qLpT&R6E;)IX37Y)a~_lKK+zOhaLf)|Ubo3=vwclH8f~b(3>7Pe
zK#gM5NRU;^gx-?AUwXQ|X-&q2QCktL{DLt44I|)0f*G?|L)aNC!0|>M!Oq(#0Fp~E-k}|67
zoQt%nc)FnExaaE*RGO?5V~Xk#WrK|**8G4G1sJj6m7CB-(s#&6z*S=M-uPh1NbX(a
z7T{#D>sCt-WWh(mM=JG^3oiAUNNn-}WlB;e93&j%ogJi%96M(p1O3@3ym!n;$OCmI
ztibk?P>&QF>>jbG2UK`(DkQARbwcZS$t9T_v!}oKXp*gl8O-0Hn=EjxrS!lRoFkm0
zM(4O-QlD+aA|7z17+1nG!ZY5}Gs=jtbB57oxCSx++dq5YTa;D4_3i2!Z4~cj0+(3xkW6~0ZYoT
zB&;H=;@zyGjP^R`6P>ZfBu~o>_db!=VzPQ`UsI3n8tV}Y_T_;6?#F%&r2K>iG13fb
zPQGMI*_QJ*W#-}BEvm^P)~%Kvq=G|)L)7IE7dYy&hXqS6u}1&sP!A=cY~8}H4Hu!OLLcd&#qTI-x2EF}l}KdNQy;F0u_
zl~-Im&oyYW6MDW&&c;+@
z>64#L^jt-#?8U$hn91VQ4LJ3{6HFgW-~6U;!Jt0Jhg~sX!aFe`>>ljidAlbgsLpx4
z8lp`!H^nT2Inr0M8j7o`M@@}2dl?I1Kye)uN5B-zj7~4nllC#bU9U?uB)X@(|GKw9
zGFgmTOX&e7*gV+0$!*?(K3yI!W8n)3@E!yRg9n3m(%{L6sB`WvIts`s7iZ)WU8dtS
zm8@`Lit3S2gS{O#x`5iwQd`5MIH9p~^w>fyrj|?@RB`nGh0|h7KE9
zz-p&hts#<|(9K<(@$^rMoaiM|TE>~_O5WbiZ81z1o^G}Dpb`un4BfPbZh@XYH;1(=
z;JrTHgO!7o`?QslQB5a(T$4w%w>)rTmj^0JRw;4XO^;?8?BuXo1&sE2Mk7E9bwV3g
zVy1-2`WS*sndS!=kg^*9lZB>REj@q)69*GFr-@ror_aG*cMABfgYRJ9VBbDx-(;lH
zN$*D1Em=#82lm-EFP&uN(Y~f0nKan9VK)kR?2|m!kSI-P+Ct2^Sb8bF(6>P{Z^3Y%
z-Zs!B3rjTF`5R2?fg`v!xV8yh+k!cLmJQ2KKy_!S4t@=O?Gt`YMk1XwYxG7iEr&~v
zxdC?|nPg=VQ&f*W8f(`Qwwr*%KEq)Ri2Q_J?GkJBvE9q{wAnU0$8OrCwM_4zOBR)G
zwe)}y%o@zvY-VjioGz!9u)+i^cZ%g;(_qs+YSUz-(MgXMY%TP76EkLXq?BaU5LZ=?
zE*fmou%-m`^^bzwk-QnEj^{n>>9Zo=p$JX#8uU!hX(sHY#RY{eT=yX
zgnXXRl%-reeK;dPDs9q{7Di^qlWkB-7LV*&OAiXcmBE$G9b^5Cjw&oB(cGd
z!H<30kICqulV*&b;H7tNgJbq%ALt@k?ZXt+BZ3B7F{}##V|{?J8vcX{eONKxdwPb=
zT4!nUI8n~78({T
zE_!2_^{N|~k_Di(lpfT9uY#`{=&KfJ=`&T>jREHR7IVQ-!BH`eN=Dh7vs2a>dc>N*pT@wAn3d8K4*kF?^?5w5qfDIfI9Mj_+(}E~nb}3-h1bFIzr{I;~m2O^1
zM!uXgN+AW~0{3<4NX*D87Otuu@iN#cVVeXP=p6$!oG}yHB%es)dU}P5Hign~UWtx*
zi|Rcv$wJPxk{+0WM}kLsut!?3q|YE>Sp?{bqbIl{xTD|Pk&I+HXN_p&g|?a3?3g>)
z2NFhBs4zwKXqLg&2-_gQJy-4_kTH2eUv$YS#~6|c!O+E}WUXbNih2)AvWRnoKs`VM
za|CnrNOQEHNuM*q#s{#|j-6nOV2he9l94MXJrV64r$?SrVCGtm|OvLz2hU;pI5g(GScLv_ldMs
zm!?#F-OCVOs>sTbeN8>GWUTM;Sg!E$JoCf%p_U0v4_yli!4`TE$)@7zv$N-1&jyNQ
zVaBet^uP;T4_wa!T+f0bU6#jVe*&oJN=5KHFYR|^B*{6mQ|NJf>mu_g?14a$6(LMf
zJ!)jI*};B<*JhXoD{(@nb8)$vSgffgzdf3k3j!6GM80$Y6(qH3u)uFAZ1Rg!aY<
zA6=v;glLtDC$ynCwf%SFEfC2<%&nFlV1dJd!x42j3wrd~8|*Ov{rv7bc$*jXHZlU_
zoUsWdL{D!Ma?9IzJtC_)7!jmLd<=FqSX1z-v_gnd>V%f&5`2!ul=dpBue8nDR%Rp0
z27_ebWi6!#RN!dfXyhEtf*gHz25Sgjo_FA7UeU|Q$c}SHrn(xE_m0U-k5q`P)ZnV>
zQ5}P=4E7AXB&9S!NfY{*Y)z*B^|VpI)0MuGOx%B(*&vWCwydS}019jjY>beNSx}?T
z!(frXYZDF(%*z=V8L@HBy;xe=YFrG(?sdh1_>ff?Oi?{rW2|{`SQhYlT++bAPv~4O
zz7`rNqf0Zi7cG|*d|);)ZsA83S#GuT01C_t%!`D1Sx}?Pxj5_tcwx4IZFw!*A|o@-
zc@{^z&1k&CdhV0Uf$Wf#7M$48BQgfN7To%OF&=4HVkWdKsb*7hwUHk3wS^wF$;`)=
zE%?a7%B_|jFo9!%W4XIySrDVouHb6_tI`a-$_sfF8F6vWsMu0McCm6FR}RF6te{|u
z>QNShZ3?dLzY>2CjO09_M~NjQOAA4ajn&c(I%k<3?_1oFg_K(@JxBtN0*`WEkFvl;
zpFzRp{FfvcxRY0LCo-DioHfamaRyhIHKqp&LsmsFMfJ#u!Nvr)?_Y*D8iIrgU5QP}
z+2G&)CD%}WD$F?PJIIkmlPkFCfe<(nIFdU$k_9LF>O!&xqiL{ELwaT*&jg5E%t=&Vtb0R1;H8j8-@h6tbd$DXK?3jCCLu
zSKwcOEC@VOnb3Y*XqiUyAG8dx)Z}~(Chfn2Y~V&1Nv_DI2S4CG;6Co+J{FMZvK|&!
z++T@J;5(k&cgX05bEd;u`s@)(jLiJ@k&ciR3tUw_x?!;Iz$Nx)XNm?MQ$9{lPtwQs
zVq&J{y!6~jN``@yE!@Zg$*qV=2
zpbl7y=d=_u+Tfg@u+BtUCR2jj%YX~76=VegQ&f*M7;GqTC;UlSp&`bMZ>C@9BZa3I
z2Wbmcb}5yT3v;t;i!rkBajT^VHDD-UC{7!S1seL?1g?EQJ8!^BJfW44Q3dCGM64mw
zcZtMoxjhmIvO<8Xsz(xxH4+9_yPuH|8eUBKetF8?v^`SZ*A!|@r82{l8-$U?j6)}d5-9m7@DIvgkiS^B>)RrM%=!PWt{
zv!9L%8d%Kuh#ZqvaHXj5Iy`q*eu|7^>c9m0Y#YdDeqWF<%H7qHl-X4
zDfRuh;}%(DLE~0S4_3e{z$<*#D=dJ}XB2RQ`bkLwF5!7xf(-MYvj~pZm0qMWh1r=B
zc_kpr_DoSdv_IG);F9zcp?^b(86VZYkO_XG3_6P;UZc`2C~iwt{v+~`vwm)KKtzH#f0o5UGyoQ_OKS88O*R7Jds6;TP;2K0QC>`
z{}J`SfIy%0$JOE|A_M4t-MU|f)lZ6kUqkTTMke+@V0l?4XNu~f^udxJmxq<3d_#sA
zA78#WlVS|T&p^q!TngO10!A!i;Yc)0JHta{H^X(Hlo&`}<3~8x)bnhqaU*P=Kt5tpAFvUpl`}
z)8m$|e&C0oS1IUai2IzHr$;|SvC%Lqc@B77maLhgdT4vBmM^%3s}s^29?bYGa7ZQ4
zd%!kS6N-1CW+tEC0EjF$tfllI0)#w-{E?7f+P+K07u>N`4*8Jq`XsyzU!T+O*3*0}
zMQqGG{Q-RobM#&_=wa)@`W;tj6@qyKgBjoErT4acrj`1|&}>Z%bUBup#oM5VEHJF4
z^uPhcJH$IH-j|;5Q|`DTs~PMe+m*?78Ja$+*$r*8s5QE}|1N$&+p^ru6xGAhgGDCf&x3^8@VNwQj_zR;&ysKuUgA(x!GWcEG*cymL4>KW`}0qYxbq(`vf~~xvB+v
zsC7+hU51~}Np)vv356{=6h^A^d@ak#j7ZQ!&V!{ouCb~@@P-3ZzEx}Kf!geBDB03T
zj8EAz(fSrSWI^FpOAi)6s6(j#i%?%0zE7p&s;W+mheTH+(Pen~oIa1)(VMc?FoUU=
z1DcklVy>zlQXVYOaT!$)csCT7^37OhVx;|%xf*&eHbxTXj5^=ohAbkirS!l6v^li-
zdu_h-dY>@IeN%s3D67w_18o0FoS%e3j%&t9R+~jGLp{a2;Y?
zff$!z;gd?-&=e`Xo9LN2^UDD%%Q7!lRSyRbR^qrGssXVZ0L=J?>Lt-^IX%6WjnB?!
zOHZk?Ybib558Vyj
zO}cw&@jlUwdz^ZZ9Ey8t#Vy0Q=j3*&#ICeQnHlz^)R`qLJ|0avN7Q=LYE?
zzNZC7-_ddY@4tQd<(L2b^7kLV`@`?P`=2lW_sj49_T{HP|Ng&z{rRsy{rUg=^RIvV
z(~m#5U-!#j|NPUJU;p{@*M8mazx?n2{`k|M{_S6Xn}2V=!QX%U^)LVU)3ws_E8l+O
zfBf_3|NikmzWnkR`U(C>?+y9Tww1@?qRTOQMrv=MLKXzpQhGWcQX5ixY41LrjhmQC
zARHolUXd-sv*#4HtC@7Pi_G-q0qx3iD^pYt%MRApxM?{tIJc&s@nO&4qowegRuo2G
zg17W>i^;Lyql7E~tbnAa?IEsTTwE``-KVT^l~MzELsm~Jt7Yi*oTiS3-qo|EF#Agm
z*jAQOnWB2gb*!dNxG6a+EVquI@qx`^opXuqMbaZZ#nC#!!mJtHK!j}fuch?#JOuSC
z3+kn@yVP{TMab!48&Y~kDJ{dQ=X5kZc1M%e^n3=hbmV|#WvP@Ys)tku3u#<*oDz{+
zzt8y6rSm>k+UZe?tu|U(J;Z#W+5&`Z@84?aX?f`Am(|frU-yY<++>^^vZ0`-Q_wPW
zdQv`H8-lB~Fw>F;^eRiCOi?{tI#@pAs^WZDOwC?qd~@;=oK4x(K)d;Ju(ia@iRTSG
z$acP6Yw2lti09W7&r4VLDQ8?qoEWVko9B|vGBkQlGv}N`NXBx@^$u87mN_{t*TbQM
zMKdlOP6orR)u(*@G5K0(Ngi#ord4_=(Fg~lm^a`c+xBZIJ-rUS{DOLUY3M$&j5~$1
zqBNB9L`qqPJI~2vYb(9anAtayF9%#IOPE|$J={51C*$JaObFZ>ea1%^efA{=634!j
zNDA2mW-ZSa8Dx9@R!dK(LnXhON?zKzPa@-T;FJIjeLRakmf_5k;_s
zWoeQrs)sMfD&vUjf71XjwRoQKDa2}Mhb?W;NyU^*FEbTx-ij^(-Y`M7-PcljnjE6|
zr9|=4$z6&#;)36NzzjJ&g&dY4%9C1{9!LnGddEaX2gE7Mj!aQKJULhl;|kv#nA;kB
z#@7fV$zXaB+2sy&#aQO8$^(8_FFAIJq|7WI$C&XySNX$Ut0b$CLB2!ckI}TRBxP>!HNK
ziWgVtMuRN%b(->3z8s>XyRwDmfrF{la}M_%5XkoUZ-w;qHKgtFI2!+^1K$OZ)aoT-J18u;5sI8*qtj9J;o)p7NEtT2hFXuB#tWmxZ=
zj&+tEcd98m=40d$3(7JbS5*(^4OX(aQZ@onsiEVPPq(F*T%dV_?4l3!Pm{tA)Jwm9
z`T5U3{`wzZe)-FHfB4-Ww-6xP+_zeKS{gF;S!L|fw|yEG_r``IE(GkO1*{C&ol~#Y
z8xu-4!S8A9azK8v1jkj?Lv@4IE3St<97|g_Px*$L%hAP>X$1o1u*pSNm@h5c{+I3S
zTP;2P4C(rm(sgOtKHZ9YVB@eBqV=JoRfglvDOMk8nI&!WVfJ@j4k%BS*0`#A2yU=o
z#a*w*f@tgH86PtX^w@7rG@DD~479AvO}&=V)5(yk&nH!vcJ0%txXd*O
zVj)r=CsJj2?VLibbW1rV%PG_Ynv>-+rl=lP8>>$}?rJ>{JgJGpj8BSPs`a5}Q$tNP
z((Y&*nJ9JB_OczlmeSM5P^V9(PM2owlBOP)u|}XPbm@b1sSKZ;6s4Iy^ZV$y@B9b!
zCd*+Qsp_G#!Lk(htsVuOt%qlP+{?ylO*ExCTVl4^80LOeec$x5?Yx%K)5DOY&m>8g
zM(xw1xK8zOScMpUgcy}!v2#k4?gD4}zE=~Ip&YQ9EOjwO^^n+LA&To#4?)be-eN(pWP2Eq8+D*U9
zcJi&3p8kdUd=B-wG-;po#GR%Gf+=*TtnQRyt&^hDMH}dSsvP$Ob--e>OvMz{Ls^3*
zCoU}sqh#ycDW9P7F4I2K5b0~XkHy;Jxwv!F>#}WZ*IIhI7lQK%1n1JCeQFaolOBbm
zkeY&0Q--Zh>P)&EUp>Vl3@=>{*h`k7xT<=%YOK<9xPBywkgaiNd<-h*l^X2@2
zvt)URDXNE{21`p^AreEz*0WPS%H(}^h8|<0kL5AYO8dgx8EVUyZcQ%RzqeX?+7`M}
zwXR%xvrkmwnvgtnLQ#q+N*QW8DJN45#uXcwr>-wNRmsv3Q&bNx4VII*2qc4v)Ua;G
zmyu|Mnx6Fx^w;Q3jlOX2_}Wc}%Qo+=mY$x4mei{ym&WW9lDO$344P1p_g9fJjC4*y
z(k6JC`;RVhi6qZZvMj{Vq8>ULEFp1|M*t05x6b%HQK4%)rX;$%W38u4Ji#+pced><
z+qt(|dKwn`QK^1hy0T9^;;xP)TtYeCSvkrO(K*>@O)V*A+ULB#jdMUuvgE@Q)x$!A
zWg~9m+#3g5t4{eyP&Dls%UZ}~k#eLRV%%KhrnhC=_J$*Rx)qvHqh?%MvQIGLx{Vl6
zLM`4?Ez0oEIjNXvNQxf+V8RKy1iT(%T!+
zGw&21Fpw$y*7*n_DYT1TeOX+D-
zC_`P!aOuT9S%}Ln0@w&mcpptDLpJ9GAzenM&>^yG#SgegmS8x>)5A4`B_Zy!+zIwt
zi%$7;O>(IOXH29wY(k`$#|>lUHcc(tu4^ehJqkUjNDnTJ*e3>YVddVa2qkz2B`Cu&
z=VTyVXRb9Q&%U&Mz&WzSa$&0K;h4cX5SLQ!fq2xQX2u6(E(AU)XHtT(#Oh<>x^~;q
zvb}n%rKdxo0=1~Xr49QeAg-O<6%e8Sdi9?Sx11Dzwp2qGk}b1X?SN}!`GhH|hgb$n
zKU^&7M!MFUGd|=Jt)^t{WvW|r;KTP;293Eihc_bolxC;D)8
z;whl?Edz(0t)lj2T>LCz^V8ubn|@)Nj6k`D+)mNYI*RXy}ESlHpJ##vzd
z;d@tSd|bpO3KK&$$@{kKuX=9L-=>LW+jA|YrzaulW|nkI!-^fe%U!t%)WiZpQ9?
z|H`)J3Ke=95+ZI|5w~<*pMt|xiax-DeEYO~lOccq?
zs!yiUj$>y2ZPT~1J-L?B(~6L46H2wE-+FYKtGLl{7UUq(J|WU%sNtMKE5!!dX2pCC
zJs=8MHeibC;f28x4fhp3gI=u(XMA>|CQ7(tpbw$GwWi~;wD(Ob+>gJ(gPtaYIGatJ
zEuGe<%y1Xs6t+Q@eN>jo(7`!P=4kXK1Y@$<$7Vb|$dbUmrXD&NtjTcI;KR7pI&j8!
zAL!NQ?CDN-q-VgAr7W8Z%(g8n+m5$ddfE`GY${c@^je=J!*znsLmTwihxC{X6`T`e
z))W4hNM9@WV(fq#WZ|DFs)q*#%Q4&-_!wqU1Lhf@UuX}p=4{Csr4XyF!84wHtMH`GOYBvCG?JC=ew_17{5c+Ep{k3#ipZLOP|7YL~
z%Ih1-O9uE)$}W>>bluYjL2Jla6E&7}hL#Ua1
zI^ci+WI_JIRMi9bgVh#B@xO{ut@EaQ9Dr6ld+J1b{)tuuCfX+(n2RxX)2p&wc&nwS
z_aL=~ORc5B`g9gX?mvcM5Lt&JO9to9DXe4)-Kh$ddCc!}0Q<7g&Q;X|@`D8y=IW2Z
ziMlS$xTOD5=y`NU%fU*bM>RvDp?dCv{`)?a?Z6ch^t2tc)mUw{^j4p+!r=S|unVe+
zS5-3Leoj)+_4E>rr3Y8`qwz;zFN^40RXyN7SW{s*{s5O+&&{|n{}O0LM6RKf=n6f3
zL{H4Jan<`SmF>S9BIxNlNUDL7YH6)LJ%#!AZ{Zfi6eFf&Q2m^eN-u8z%GwQSf^^vuRM-)>}*I={HEIQ4(rt
zsy_XM5%#ab6~xoOiYFN)Kc}3Wr90q>Rx~gd!4HsK7QUIHdVqYaaLO1`-ysq8TAXo>
zd`%@-QLFl_{r41%9iOUXv-
z4EL@skLR+Oy|1YUy9X;J447~6r#0D(+uURH#>PN*_b6AYsg}a%r1xDZ+i}-Qdio3+
z>G2wAX{J7bgz51EfPy-@sUsQGJ|~T=r^jn^jh>lFJwS0;tY(Vpf$hQa2;<@J(5H3T
zjEmJBy*^lyH8eD2X|KBVj(awG)0VOwcY^>ujRt-6V12Z7Q=d4(1ULszP)5HgBN@Ox
zDT`_`t_I7ECLI8{EJ!m&^`P}&S%eAiEACK>g&7x|yF$-C(Fg4|i&Sf-J96Bvr%g}F
zwwqmR>FF|PqDN|?rIq>w5k|Oozyvka)Q}8LpOr%7Q0W45Wo+sJZp&gbQ&bN~50*lh
z(SAdm)?ZUD8jmH`O3xo#`fg^7w}qMSqxIwO8dA2`ZngBZ7;{
z1KM<>Y@^X5E5CKVp8kUNd5rd1ny5?oBn(r(1EW`0Ju;|#PV(erD_wG7c5)wJv@8xY
zMfCvkV8Qe91JkXqrrf#hou{d2L+_eIT3|%16`6eXeJ9E`*|n0M&Vtl=fYe#qr%&g+
z{7m$UK`$+G*8dj%`)^-<`Q<;q{Qbx8{_wl+{^!g8{qp<2efjCnzyGgafBx%FfBrxJ
z{OjNT^yAOp5f1(G*FXRC<=21y{Iy^A`*R8>&{LfuvCpVq4uDw}ez~f8pm?yzdF`=h
z>MK9x-fK?-bcM!wg10o(pMovS!qiPC%J$e=N>5`!<49|qrF;4W&TG##ulV!2>Sq0~
zU;=4FPbZm>s^us(K)Ju(o;O+2z(%Q|_NmE>b#5`xAqy^!jWl+-8Yw
zAIf&vt(Km?g0vBpHcRvL>6#ZFPhPR-MMcf}=bu5*7`lgLO{&a-+RFhJ%i=9pRS)tG
zRy40Vg4|ka#zoM+x=cdGCVHW))RbuWct7c8H$5oZUAJ0#dJ1YrPR%Th(mPms5#uP3$@GuP3^g7guq?_lMfJe#SP2tfa*UX|$<4TunKmHS>XN4&R=Id%
zDE-{)mu>sWcGj(yo`!;c5z;S9*K~=O_>zOdE8M)Ca#{c66UderTy=)rAp@yA?h0e;
zzNj9M9jsbjZx*<9(v0ho3+?|)CBCuccJhKlTJNrCl41mCPS=hk^%NSw!8})B~-9RmzJE^R_OUaZB-AGbMT+)cUrp
zI7DXs!M5vUJL*L{SP0Pu4(^^VT4?&IGQH?B((kDe;W>j`Xm6y;V>z{T45mJ3}^wFCc
zDmg+_S=i*N>Os)4DkQwdFl=j|8Mo$o67wO`_^S^&(L1iRsCutJ-uIhqCtWM)=^p5h
zd+Cp*MY_aCc#Tol6-=IAd8~ib31mkKl^mKe%$q6)Fe(d{Oi?|!IaqbPz%XlTof%ix
z#^N(==`V@Cz*rmUf0?PPO|Qwe(OODR^FVamMRY73(x*6HU|w|vk|&oN>z{K1twHxx
zC=#Zr)V&)!z)x9_WQyv6%fVXX*+){TZ_Jb%WUY6GF38Xg8G7*0mf|fF8E!gFxQVV!
zt*3RMH2Rgs(jR>?;N)P5@zkTG
zt!-vplm$I116xD-{gGI&j&WE;^nQ@ouB^Y{0&$YGiG~X00J!s5)8?l=v
zlkJ~dEj?`mQPCqRmfq-76wf;Yx+2GO%8B)lH-VOL#s>;#4R=HD2q{QUduzo@_*NMTjSo2kW0{0&!sJoz`4}Wmi-k
zpr9<&aoDQ|^#<#Mry7K$J}FZUEVxV;l2Y`Qe!dh<4wli@+s=~hnOiMAodSLENqw-i
zL!UZ$s&U5^AD%`Vtbd{@lmR_FQ0a^^*mgNUKv|UIs_MbJvC6r)1mZ
zR!dKpKn;9C4J@tDrv;v79C5{kr%(dxpJoOfkP7`X(bI(cIqM_TlSMYJsve9RtOTB9
z@UV5rjN^u+mnkeG8Q|!aW2Qyum6HLRmXd82yVlavB9H+emH|s2^of8c88TcU;Rz(b
z`e&Iz0AyN)PoF?Nv!nI^?qng2DXItC2GjpjOcb{6m~xJ=z`x>-Y>H)N|bZ^zq|Fzbba_X-TyT?NNKa+7Zh@Y8M
zY|~1zO|q8K(;1-p4^aJb{JV(Wsvq=QF+erVuYZaONZwhOlB312%=Y%gBAYCH?Q816
zvB3vc3Q&bOx4Q6-MV|uMAW*p9In}g}fo=+wb%hT@Qz?k<<
z7YR4WwPE$N1(^LC%)XrcK3-QnkheC!T2^2G1T%2DY2Out^$e*Wz?m#&F-7&D*I-gt
zJbc%BV#bj>M~g0_qoq74(sp0(oZ~2c(?qiUK~E$8*06fo0;K*`QeVz~AEhgvvs+tV
zDWR`F{{(dItnsE=is1J{bq7!;3s+20J-9WP&ee|8QAea12kBgl(T8GUa4C3SXmvEx
zCYu%#?vAyVo~8hqe}&AKlix?>YUk(HwpU5y>rX!cjc2;&Qwm-1Vc?0!Fkzh97u5q;
zV_DoNos?_sFynmOC6w$cJwQQOJ1xYa>u`zDx|;@)?TxjRo{j*8A5!>o?7IluD4mO2
zJ6<7uuRr?)&ds_H?h
z!K|%tysh=Zj6-dg&XND%}^d)>&_Zjh3Fcv&GWP6fOOr2Sn&15z!Fca4MwJX#c)<-uvJq9gWSE(g*pBZ2akwsJgFdaB&gg(x2b%><^HiOgtIS)%MZ(Zgi4os?*eo&>3F%Sm?G
z=YPSg)&^6~m8DFZ)nlchOH(|}U(z3RKYh7r9NCswOX=wYkTZMOd#IUKX{qdPHN-4~
zN5B5#zkdDsuRs0y-~RnO`aj>_fBf|?|M=5?e);8%!-U*DdrJbcY#vGXSD3=3llEojcsvaa7%*blS
z!def^I1JW4=v1=F?fO+L6fFeiisH6uWZU6ZOHUJkj^EMoa?<QT7
zAmWfcEno71amELjB#S#tQ9S@On1z*$e6{00<)D{q5j<^J$>uGgZyEnSFWNMWY%i>(
z^mG6yc%|UwnD-H|lJTy!qZQEa`m;_zzP6Iir;meP#`EDCVwiRx*^fUnI_c!`oPI({ss+iqc+j)BGU4POU
zh?kZNd74QnWsfljKqLz`Oi?|kGnRIXf-$Xj_GcWIC4d!-}GIVX2cB*fYZ8-Ok?nxBl%8|{^3=K;NW1dK|Hr)Z%(J(08rkt3*)g%hr-9$*(BZ(=ryHgKg7lBGRb%o4~=lgJ+bwUnMS|I%c-9PBPC&C^aWtzA1M
zk*+`81T;#U{c1{-aRrV?d5n<-5)Rq)V9H<~opfNSo%k8YlrBV}#GaJP7<#pfa@OKF
zQ@ZI9+0%dPM?Gi$g?V&2*nKQI>3q`KvlDXY`m@czptJ>-kb`i9VbBA7k;M=W)%2jr
zUo
z8o7ahO=rlS`n8mv!~UXtxt!=eww!RFXl>NVxN`luCSXdE)`12QIc?3ui7aL?7^Vk3
z22xQBG!HHPe&zqX*BgZ`2D{
zD02O&CLl;#LVmjn$9%jw01{caV2bKNj-lMVi?JvtFXu0PWZ{74r%Y5AGu?y+4Ca6}d>xT<r+wPIDhQPBqW7
zw&yg|xc)>F5F>qnv9w3UFt;)f@Iw|On4)^XVlXRCG`Q1__>AK^mq0H;*BU)tzclpX
ziz|sqRX0r`d*0VldQSO^QQ~rx`v`HOA)U1|C!xdj=b3;E9nBazpNeC0q$A{zg$J&x
z9;6t|hto{uv@<^AEKVquhPI2d^mUQGX$N)`ZQB&G$Ng4I&mn&)I$Tb29~n+FgR{2f
z6jZqWG!qbE_4JPvxGRwdm?4V`Oi?`mF_;A>8MvY2ZD$;_xzL(B3P|Z8vXmli5ircA
zyG=XDp7gbpp7Z@e1h^dHKKh$vx@PUi3CM5#Stg*qRE%@hJc!oB+(!f}t2TELx#wB_mWdC(=(+si)eJ!Que7_9wEr+;^_Tm%+Gix^v
zC%yG2nSk=5PcG5?95WtpIe-aSEa0l@frY`eH^FR7JKQr)#9Um;hMuM~E*5$b*3pH1
zM&@nWLH3}prSzQc7a_gn2=`Im1YrXKQ(N$ACU2cfXrN9GlkOcsys2(I3Oml;WS=zatafF3lL8XDtTuIImy?AB=xZQMt
z>;b>k(sQm~fZ>+I+sAK%r&iYf8^~_!&oBYE0V3eIo40lY0KH|W3-`zC~NZ#;m(WVL7%0v^C`xZaI0oi6}it1_m!CW?MmZTl%
z87D|G9d27}G9??D5jKh2UcKo6*~7h-(sQJrn#q=P+s9+W#zxkr8^U61^$9qP#@AwX
z(MOKNcnFZ~a|U!uo@O6RU_(Yf+Hsz7;NucQ
zwXyg_n=M`TvDCnXiJJzHJ=$w2J?Hsp$!j^ZebhB%&|~ekhZEOYc>>zXfi9!cE+S?n
z-%;Dk_BK~lPn!>>tpURwKYU+joangFZIh4^y%9@Wn4PC_eD0ap&FPmt*J~*~hxti4
z>+Qt$5!Qe?j%%Min60j5C!i|pv#XUp1u)|CsNH3Inyad(y~lEufB4V_9pxD(HKH-?
z#kmqKjcY_&hUJ+EVSDyvPxY;qp0oUvRP}aXyNJp^d_d#cWDljOYtb1<%2ex1tlYyN
z2c0h4%}h}}Z9SNx9z2Ud$9Tp8j7xH!K3ydjtc~;$i@szrS3vCM;LD!rTP;0D`3V{7
z?Y#E!(}M>ut{wJ3cDk0Fftx6~r`34$&;7^eqb8SaWUi{7mLAMb4;`|gV?5_%McZ&;
z=xrHKH<2wZmrC4yrhOdru%U-*OFe{vuEi$cA0Gm}
zkn5OHyn}9*ZCR$Mp5`4(Kkh-p4Rm;C9BIg;qm1>G%F$Fq^v-fGKEFThvIqHENzdth
zBIyP5u6-m-HrFx}P>rQKfYBKn*_k)KePw&}!hnXJb{)($;^PH$bY~nY
zNc7r9AfhqG&~imv=>d*C(QMDU>?yw0(sOp7f@pZ1aOn9CmEwUnMytDaZhj%F9DSkWnfYwL({%C*1*j6x5c
z(IVo5K{LKrg&T5TR8MOTCKSOh{&Zw#{LXKqbCsTfD1{d17<#ZTbK3|uXI%F9-ug+;
zsZ~xWZ%4C_P6R*hUmHh=Os>EE1XL0O-7Bb--Dc0XscK+sz4=J-llvJx5kGjl7-9J`xf8WPj}$5em8f<`WQzrHAY2<2=3l
zv;UlVIcQMXF664}X~@AGBJ_El4(yB{E&kdu0{n6Pt!H2llKYk(jWbN!9W|$H19DaMwBlg)5c%@{!}n>*@9nhn
zsFp$-CmeByaWp*4eBoCh4mOy(7xZv~Lc#?6F--
z={c)P`Qq(BcCm#K_E@4z}!>T}$aXsVZ6G?Kt*v#N9uMUt8p^3~~KUC*TL0
zQ%%`$9d^)$vc1L>)zfu@`Qgr=zv-0D_|cnIlF(z$q4|eG>xt>Z1rwcaj|dXCaDi%jh1SPP%nwU(ZPs*V}nPGcW0-1YnQwL9*{3fJFo0#1mI
zM%80r2b~VOPqxD@OjSL-HkcId_^JAb@8gUgs8dWXW!gqfR-s)7x7=#0&7qb(u4^eh
zXH*#1$iui3_g3-2_ZP-5PQ;W%nJ*|y@U>S?gSEO4if%XC0z{G@!LqZ&-jw7A3OHhh%0_dIAR;kzR(
zdsuI^^c+z|9PoAy`xxL(-;%E#aR>ar{#G-ff39@*w3fi#dOK(~*@og;Tu*BaM*q8f
zI;P_};|JqZTrxG#=GkCe&F!`{BmXz&S@xvbwU(aqsRsJr4q+eg-{nK`wHdl$|N0xv
z0Q&SrI0X8>#x05EyG*u|nAX#7V=Q|>*HU^;<@C7zcJ%sCz3U6?_2{3*^y_ah
z0ix4u2cgj5T4KIj9rTiH!!Sklw9a5a@A%eAM{>q5tfB3ot~3!$VSBaogfur7ygkFR
z$MaT8	ao(BF<;ADnl5SG^wmlVE=R+h>4sTj-rl`Y6Rf`9UAa_6k!}Pty#B^3&f?
z=|Il-<@6G3qQ^7a7k-M=eWG;T?S;AB9AVkhd8?)8P)?2HZzr!0$4`GSy`K70D1QB$
zXTb2}0_hzWnCnkREhO6}Tva^{GZ@8Bejokedq3k>QF@IcSm){EPE52&Ez=q`2EsQd
zSoUn%wU(X}IWdI49lJgNKlwHEdf-pM_w{d`0lXu1hzXHfo_x?dvi-pn)zc_rp*x@Y
z@=1qr#_yek*xvXO3_ZsZXl*M6wM)zcw^5&OjFOgf1(e#E?xz;01@@X@w^wUzUOo3kr>
zEZ0(c4&$V7{dVH|!1~0u%j-G+BvfDjwi$q$_G@?(BLmb&-6Go(Tva{YF&L^p{h^Xh
z;*6gsFWwb;V66t9=;0CXi}TFRit_&K%AU%#l%AtFB~ZVew?3Hu^f$@t8UGYWU;m~F
zAlmwvv$H-hso+tg$aVu)RZn9K#^_Iefuw^t<#$Kg&Peyo=y52zaz!gq&o2keXLX@#+noIdfzkdEMt-w7`{B^xYd&vea;c3RcM
z&FB$K-_ES?aa?QZIfat}^4n4Cg5&gwuY=cv{XsCk{_+{1*v3jHK5$Tc&=<02pDC)R
z2?j&)hd%|<0i5y!AU!~5OtrLAwos@{8*}*AvfdwA+0$s(_Rw<(r$XYllh%jBAO7%n
zJ=GsV;p;D+0E21rnVtbk++!-2gN~3r^;}gw=YKE=fAAw8oxd4B?WIy24LVyxAMa=x
zJx$~@T6l9{Wsl-oO3(S52!G!WSs(g-@O$3%IDY_pufKExD$3z8$eX%>B?8
zy6aK?CgNUy;RJYFe2nzYcwyc+IA{UcbIugibK(c%?FT-^{qVh=@dF%f&uXui(R&;8
zgeC1viZs8rhuh6rl|6=QDLsd861aUkVSQlxfv;}YbNmgcz5cQZfR+SAHPz7lr~Sb`
zI{va}oU5wmxR1r!_|0!@bo6HYvPSR2L`$2bEd>fb(em?yJx1@&s_ZGe^@X0ZHwDzb
z9k4E-joJ^_H;8v^&IoT5c`d9XLRhQ{9;ye
zD1{zbjGi9kN~S#`y=VOGL6tp&w_19R-UJBycE0-X^&8*Gt|#~_=z9Gn6JRTaV1}L=
zvfM(TgEKFCt}jegJturHxPJAU7@fKqzl6~)Wx6(DO`@l-XbGn++yl3pV=8+9*HU^;
z-EdreJ6e6H`ql4W*Q0xgsn=gH0ixP$DegGY$+~iY@d%&F|Oh=+Z*S%!-iB8I?VLYbia4ZZMX$yEZ((8Xb0g#f*qPGsJGt7%MN5@?DEOS-$9P6=I8je3z{qTLAP?lcW%CQ_hMW}^t
z0p@5OGx54Tq_W5FR!h&J8;hmwWOX5FIQ};E+o|PoRN_cqCFkelsHG+^5m~C8&;|-KVE5}I`IZ-y_UH|s)kMYb52e&?zQ!%xhaBlK2
zHMHt<(*(%d$tGsazSgCnWoRU?}wNZwL4@n&eO28dZXyUcDP+bq30
ztHa^AXXVuUkH#n79v*F~lXML353w*M-r%Iple-Ag7M^Cod$2Ve-1?JE#nf)9srMN*
ztae4y1jO5ECZ^a9(9fyORHcX7EVx~p!vVNoy>zrbA}mL`hjK|>JD`;MsF@a~rt&nt
zp*EOip?k139N2nOOGVU9V(DnzLJg=Ltuz7fc2Y?ltXMhU$`Y7<#0Nc8wf_&eFxhwU9K-+dcim;jAx-R50zPkUFnVgXp?Injm*eeZ-9C
z9v0Q|HJ6MuB%8%`&ERmL?O7Z(P9Ds3vxnM9U3BVT%?IDdEKfSWc#B4}u+iVPhQnD;
z;;3MHtTsxq036UXTjm$=F2-;N+#h!QI=PqG>y!{^-190$1w{Ls(Sz&u@-rtN4`mEbVu8=*La6G?S-!naVWJaNTd3
z02|efK@T*Zvvly;%Q5nYHEEN;#<2k>K
z|8`V?f$F#8uRorw9(DDs$n*I`5m{G99Jo)x8d(uI?#PKED+)ME{KSo|B=q9CI^c82
zuAVx?CN!)U5%~rAks31q

literal 0
HcmV?d00001


From 24f8ca39fdf3034e82ef8deddba15821eba9f20c Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Mon, 10 Jul 2023 16:43:38 +0800
Subject: [PATCH 082/161] fix: max recursive error.

---
 dongtai_common/engine/vul_engine.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/dongtai_common/engine/vul_engine.py b/dongtai_common/engine/vul_engine.py
index 63c274e6e..486ea9de0 100644
--- a/dongtai_common/engine/vul_engine.py
+++ b/dongtai_common/engine/vul_engine.py
@@ -134,6 +134,7 @@ def search(self, method_pool, vul_method_signature, vul_type=None):
         from functools import reduce
         import networkit as nk
         g = nk.Graph(weighted=True,directed=True)
+        # Gather data
         source_hash_dict = defaultdict(set)
         target_hash_dict = defaultdict(set)
         invokeid_dict = {}
@@ -151,6 +152,7 @@ def search(self, method_pool, vul_method_signature, vul_type=None):
                     lambda x: x.get('source', False) and x.get('signature') !=
                     'org.springframework.web.util.pattern.PathPattern.getPatternString()',
                     self.method_pool)))
+        # build a graph
         for pool in self.method_pool:
             hashs = list(pool['sourceHash'])
             vecs = list(
@@ -161,11 +163,12 @@ def search(self, method_pool, vul_method_signature, vul_type=None):
                            set()),
                 ))
             for source, target in vecs:
-                print(source, target)
                 g.addEdge(source,
                           target,
                           abs(source - target)**1.1,
                           addMissing=True)
+        # checkout each pair source/target have a path or not
+        # It may lost sth when muliti paths exists.
         for s, t in product(source_methods, vul_methods):
             dij_obj = nk.distance.BidirectionalDijkstra(g, s, t).run()
             if dij_obj.getDistance() != 1.7976931348623157e+308: # INF here!

From 99176ae1fa2396ffcf5ae4b58bf615f913d4cf52 Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Mon, 10 Jul 2023 16:47:44 +0800
Subject: [PATCH 083/161] fix: max recursive error.

---
 dongtai_common/engine/vul_engine.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/dongtai_common/engine/vul_engine.py b/dongtai_common/engine/vul_engine.py
index 486ea9de0..fd11f3240 100644
--- a/dongtai_common/engine/vul_engine.py
+++ b/dongtai_common/engine/vul_engine.py
@@ -133,7 +133,6 @@ def search(self, method_pool, vul_method_signature, vul_type=None):
         from itertools import product
         from functools import reduce
         import networkit as nk
-        g = nk.Graph(weighted=True,directed=True)
         # Gather data
         source_hash_dict = defaultdict(set)
         target_hash_dict = defaultdict(set)
@@ -144,7 +143,9 @@ def search(self, method_pool, vul_method_signature, vul_type=None):
             for t_hash in pool['targetHash']:
                 target_hash_dict[t_hash].add(pool['invokeId'])
             invokeid_dict[pool['invokeId']] = pool
-        vul_methods = list(map(lambda x:x['invokeId'], filter(self.hit_vul_method, self.method_pool)))
+        vul_methods = list(
+            map(lambda x: x['invokeId'],
+                filter(self.hit_vul_method, self.method_pool)))
         source_methods = list(
             map(
                 lambda x: x['invokeId'],
@@ -153,6 +154,7 @@ def search(self, method_pool, vul_method_signature, vul_type=None):
                     'org.springframework.web.util.pattern.PathPattern.getPatternString()',
                     self.method_pool)))
         # build a graph
+        g = nk.Graph(weighted=True, directed=True)
         for pool in self.method_pool:
             hashs = list(pool['sourceHash'])
             vecs = list(
@@ -171,7 +173,7 @@ def search(self, method_pool, vul_method_signature, vul_type=None):
         # It may lost sth when muliti paths exists.
         for s, t in product(source_methods, vul_methods):
             dij_obj = nk.distance.BidirectionalDijkstra(g, s, t).run()
-            if dij_obj.getDistance() != 1.7976931348623157e+308: # INF here!
+            if dij_obj.getDistance() != 1.7976931348623157e+308:  # INF here!
                 logger.info('find sink here!')
                 path = dij_obj.getPath()
                 total_path = [s, *path, t]
@@ -179,13 +181,16 @@ def search(self, method_pool, vul_method_signature, vul_type=None):
                 for path_key in total_path:
                     sub_method = invokeid_dict[path_key]
                     if sub_method.get('source'):
-                        final_stack.append(self.copy_method(sub_method, source=True))
+                        final_stack.append(
+                            self.copy_method(sub_method, source=True))
                     if sub_method['invokeId'] == t:
                         self.vul_source_signature = f"{sub_method.get('className')}.{sub_method.get('methodName')}"
                         self.taint_value = sub_method['targetValues']
-                        final_stack.append(self.copy_method(sub_method, sink=True))
+                        final_stack.append(
+                            self.copy_method(sub_method, sink=True))
                     else:
-                        final_stack.append(self.copy_method(sub_method, propagator=True))
+                        final_stack.append(
+                            self.copy_method(sub_method, propagator=True))
                 self.vul_stack = [final_stack]
         current_link = list()
         if self.vul_source_signature and 'sourceType' in self.vul_stack[-1][

From e183e99efa58ef1360cb4cfbfb356fc2f90d6188 Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Mon, 10 Jul 2023 16:56:48 +0800
Subject: [PATCH 084/161] fix: max recursive error.

---
 Pipfile      |   1 +
 Pipfile.lock | 463 +++++++++++++++++++++++++++++----------------------
 2 files changed, 266 insertions(+), 198 deletions(-)

diff --git a/Pipfile b/Pipfile
index 07945d21a..ee23e0b10 100644
--- a/Pipfile
+++ b/Pipfile
@@ -85,6 +85,7 @@ dataclasses-json = "*"
 django-silk = "*"
 types-python-dateutil = "*"
 shortuuid = "*"
+networkit = "*"
 
 [dev-packages]
 
diff --git a/Pipfile.lock b/Pipfile.lock
index 904ceb77c..034b72ad9 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "3ee3a2c2024fcd3b31ff46878e1d417463673a40bddd822b682fecb9936d4e20"
+            "sha256": "4fe4b731e86a03b2e7707d5996956c116300d8fe2186d79397b1ae205e46fcd3"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -101,11 +101,11 @@
         },
         "boto3-stubs": {
             "hashes": [
-                "sha256:451749fc2bb0af5718bf1410473ec2e7f915bb860614cd0f6aca00c254ccf7e3",
-                "sha256:55b094ebbefecb0b8015451707aafeb81c2313e216dbbd5e2f2efff70a02db63"
+                "sha256:4595eebb4a92808a26b827db31e4ded3d1f9caed64c539d468c4ff2f327badc2",
+                "sha256:5fb848352946deee89f260f062f8a8238b282a39e5bf36edf8d79aa72af2a143"
             ],
             "index": "pypi",
-            "version": "==1.27.0"
+            "version": "==1.28.1"
         },
         "botocore": {
             "hashes": [
@@ -227,92 +227,92 @@
         },
         "charset-normalizer": {
             "hashes": [
-                "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6",
-                "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1",
-                "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e",
-                "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373",
-                "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62",
-                "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230",
-                "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be",
-                "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c",
-                "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0",
-                "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448",
-                "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f",
-                "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649",
-                "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d",
-                "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0",
-                "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706",
-                "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a",
-                "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59",
-                "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23",
-                "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5",
-                "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb",
-                "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e",
-                "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e",
-                "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c",
-                "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28",
-                "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d",
-                "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41",
-                "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974",
-                "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce",
-                "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f",
-                "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1",
-                "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d",
-                "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8",
-                "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017",
-                "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31",
-                "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7",
-                "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8",
-                "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e",
-                "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14",
-                "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd",
-                "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d",
-                "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795",
-                "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b",
-                "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b",
-                "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b",
-                "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203",
-                "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f",
-                "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19",
-                "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1",
-                "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a",
-                "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac",
-                "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9",
-                "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0",
-                "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137",
-                "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f",
-                "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6",
-                "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5",
-                "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909",
-                "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f",
-                "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0",
-                "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324",
-                "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755",
-                "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb",
-                "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854",
-                "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c",
-                "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60",
-                "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84",
-                "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0",
-                "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b",
-                "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1",
-                "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531",
-                "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1",
-                "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11",
-                "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326",
-                "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df",
-                "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"
+                "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96",
+                "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c",
+                "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710",
+                "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706",
+                "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020",
+                "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252",
+                "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad",
+                "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329",
+                "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a",
+                "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f",
+                "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6",
+                "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4",
+                "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a",
+                "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46",
+                "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2",
+                "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23",
+                "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace",
+                "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd",
+                "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982",
+                "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10",
+                "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2",
+                "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea",
+                "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09",
+                "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5",
+                "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149",
+                "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489",
+                "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9",
+                "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80",
+                "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592",
+                "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3",
+                "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6",
+                "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed",
+                "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c",
+                "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200",
+                "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a",
+                "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e",
+                "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d",
+                "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6",
+                "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623",
+                "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669",
+                "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3",
+                "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa",
+                "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9",
+                "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2",
+                "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f",
+                "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1",
+                "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4",
+                "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a",
+                "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8",
+                "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3",
+                "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029",
+                "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f",
+                "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959",
+                "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22",
+                "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7",
+                "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952",
+                "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346",
+                "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e",
+                "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d",
+                "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299",
+                "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd",
+                "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a",
+                "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3",
+                "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037",
+                "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94",
+                "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c",
+                "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858",
+                "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a",
+                "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449",
+                "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c",
+                "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918",
+                "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1",
+                "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c",
+                "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac",
+                "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"
             ],
             "markers": "python_full_version >= '3.7.0'",
-            "version": "==3.1.0"
+            "version": "==3.2.0"
         },
         "click": {
             "hashes": [
-                "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e",
-                "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"
+                "sha256:2739815aaa5d2c986a88f1e9230c55e17f0caad3d958a5e13ad0797c166db9e3",
+                "sha256:b97d0c74955da062a7d4ef92fadb583806a585b2ea81958a81bd72726cbb8e37"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==8.1.3"
+            "version": "==8.1.4"
         },
         "click-didyoumean": {
             "hashes": [
@@ -1142,7 +1142,7 @@
                 "sha256:70e991dd503e3f1956632c621946bb8e71012172db740bab75cd9d2a447283a3",
                 "sha256:bfa5d118f91f08ef4694d4aaab168c98efc91dab086aee587e71d66a7001701a"
             ],
-            "markers": "python_version >= '3'",
+            "markers": "python_version >= '3.7'",
             "version": "==1.12.0"
         },
         "mypy": {
@@ -1198,36 +1198,58 @@
             "index": "pypi",
             "version": "==2.2.0"
         },
+        "networkit": {
+            "hashes": [
+                "sha256:0ff615156110510c3c44d538baf6f48caf06af0e0b9fe4e4e3a1c527d54598b7",
+                "sha256:1b223201f63d0f282af75ff44710d476b414c6229ad767b6ca9242c9d25f0abb",
+                "sha256:29a31a1ad709f802d113907ee47f481dee713b0a72ebe44fb0930e3a92be6536",
+                "sha256:34e42a1a6836149265478bb0a70e75382487188e29632811ee894e231fe25788",
+                "sha256:36caba5c75bec9dab52a2fc98b6f24ebf89ad061256051645d73c71d88c1fd92",
+                "sha256:45863a9a1d554266112cfda895c5bfc0b9c74a80ffbb1b4bfb9fbf5d03533bd6",
+                "sha256:5b56564385fd4b749f34a2745c06b06378e124c46082342bbe9dbf1a73c9d76e",
+                "sha256:67b71732595ee1f192d1ffa93420087ce0fef63bd59cf0c3484e30847c997044",
+                "sha256:842781097d8d6ee9db2d847d9d7de958e36cf3d4c20bf84db7ae63cae12d7479",
+                "sha256:85071cd39692e93b3882dfae642a71421d4f74bc40ecede3877272c945e4bbee",
+                "sha256:9282d27edef5d9376b3016c047829d696ced67566b76f79679940687f27aaaab",
+                "sha256:98acc4c94b19353ccb074e689dea91272e877c4fd6b60c69ee8869433f14a76c",
+                "sha256:ae4e85900c2eb682ebef31b6bed3c645946394c4490318d10a3c01ed12d46bdc",
+                "sha256:b0cd7134c37aaf0c4c1ab489a0bdd8cb53d369c37a2b8469c6ce8be0f270faa4",
+                "sha256:bbb523c1e00aa90e35fd29585c55ee2a2c58016e2764e2249b8ae218e0678fae",
+                "sha256:bd0b62b1fcd656a8d33ab677aea4e14af1a8b53b235a2316c88f7ee79e6da784"
+            ],
+            "index": "pypi",
+            "version": "==10.1"
+        },
         "numpy": {
             "hashes": [
-                "sha256:0ac6edfb35d2a99aaf102b509c8e9319c499ebd4978df4971b94419a116d0790",
-                "sha256:26815c6c8498dc49d81faa76d61078c4f9f0859ce7817919021b9eba72b425e3",
-                "sha256:4aedd08f15d3045a4e9c648f1e04daca2ab1044256959f1f95aafeeb3d794c16",
-                "sha256:4c69fe5f05eea336b7a740e114dec995e2f927003c30702d896892403df6dbf0",
-                "sha256:5177310ac2e63d6603f659fadc1e7bab33dd5a8db4e0596df34214eeab0fee3b",
-                "sha256:5aa48bebfb41f93043a796128854b84407d4df730d3fb6e5dc36402f5cd594c0",
-                "sha256:5b1b90860bf7d8a8c313b372d4f27343a54f415b20fb69dd601b7efe1029c91e",
-                "sha256:6c284907e37f5e04d2412950960894b143a648dea3f79290757eb878b91acbd1",
-                "sha256:6d183b5c58513f74225c376643234c369468e02947b47942eacbb23c1671f25d",
-                "sha256:7412125b4f18aeddca2ecd7219ea2d2708f697943e6f624be41aa5f8a9852cc4",
-                "sha256:7cd981ccc0afe49b9883f14761bb57c964df71124dcd155b0cba2b591f0d64b9",
-                "sha256:85cdae87d8c136fd4da4dad1e48064d700f63e923d5af6c8c782ac0df8044542",
-                "sha256:8aa130c3042052d656751df5e81f6d61edff3e289b5994edcf77f54118a8d9f4",
-                "sha256:95367ccd88c07af21b379be1725b5322362bb83679d36691f124a16357390153",
-                "sha256:9c7211d7920b97aeca7b3773a6783492b5b93baba39e7c36054f6e749fc7490c",
-                "sha256:9e3f2b96e3b63c978bc29daaa3700c028fe3f049ea3031b58aa33fe2a5809d24",
-                "sha256:b76aa836a952059d70a2788a2d98cb2a533ccd46222558b6970348939e55fc24",
-                "sha256:b792164e539d99d93e4e5e09ae10f8cbe5466de7d759fc155e075237e0c274e4",
-                "sha256:c0dc071017bc00abb7d7201bac06fa80333c6314477b3d10b52b58fa6a6e38f6",
-                "sha256:cc3fda2b36482891db1060f00f881c77f9423eead4c3579629940a3e12095fe8",
-                "sha256:d6b267f349a99d3908b56645eebf340cb58f01bd1e773b4eea1a905b3f0e4208",
-                "sha256:d76a84998c51b8b68b40448ddd02bd1081bb33abcdc28beee6cd284fe11036c6",
-                "sha256:e559c6afbca484072a98a51b6fa466aae785cfe89b69e8b856c3191bc8872a82",
-                "sha256:ecc68f11404930e9c7ecfc937aa423e1e50158317bf67ca91736a9864eae0232",
-                "sha256:f1accae9a28dc3cda46a91de86acf69de0d1b5f4edd44a9b0c3ceb8036dfff19"
+                "sha256:012097b5b0d00a11070e8f2e261128c44157a8689f7dedcf35576e525893f4fe",
+                "sha256:0d3fe3dd0506a28493d82dc3cf254be8cd0d26f4008a417385cbf1ae95b54004",
+                "sha256:0def91f8af6ec4bb94c370e38c575855bf1d0be8a8fbfba42ef9c073faf2cf19",
+                "sha256:1a180429394f81c7933634ae49b37b472d343cccb5bb0c4a575ac8bbc433722f",
+                "sha256:1d5d3c68e443c90b38fdf8ef40e60e2538a27548b39b12b73132456847f4b631",
+                "sha256:20e1266411120a4f16fad8efa8e0454d21d00b8c7cee5b5ccad7565d95eb42dd",
+                "sha256:247d3ffdd7775bdf191f848be8d49100495114c82c2bd134e8d5d075fb386a1c",
+                "sha256:35a9527c977b924042170a0887de727cd84ff179e478481404c5dc66b4170009",
+                "sha256:38eb6548bb91c421261b4805dc44def9ca1a6eef6444ce35ad1669c0f1a3fc5d",
+                "sha256:3d7abcdd85aea3e6cdddb59af2350c7ab1ed764397f8eec97a038ad244d2d105",
+                "sha256:41a56b70e8139884eccb2f733c2f7378af06c82304959e174f8e7370af112e09",
+                "sha256:4a90725800caeaa160732d6b31f3f843ebd45d6b5f3eec9e8cc287e30f2805bf",
+                "sha256:6b82655dd8efeea69dbf85d00fca40013d7f503212bc5259056244961268b66e",
+                "sha256:6c6c9261d21e617c6dc5eacba35cb68ec36bb72adcff0dee63f8fbc899362588",
+                "sha256:77d339465dff3eb33c701430bcb9c325b60354698340229e1dff97745e6b3efa",
+                "sha256:791f409064d0a69dd20579345d852c59822c6aa087f23b07b1b4e28ff5880fcb",
+                "sha256:9a3a9f3a61480cc086117b426a8bd86869c213fc4072e606f01c4e4b66eb92bf",
+                "sha256:c1516db588987450b85595586605742879e50dcce923e8973f79529651545b57",
+                "sha256:c40571fe966393b212689aa17e32ed905924120737194b5d5c1b20b9ed0fb171",
+                "sha256:d412c1697c3853c6fc3cb9751b4915859c7afe6a277c2bf00acf287d56c4e625",
+                "sha256:d5154b1a25ec796b1aee12ac1b22f414f94752c5f94832f14d8d6c9ac40bcca6",
+                "sha256:d736b75c3f2cb96843a5c7f8d8ccc414768d34b0a75f466c05f3a739b406f10b",
+                "sha256:e8f6049c4878cb16960fbbfb22105e49d13d752d4d8371b55110941fb3b17800",
+                "sha256:f76aebc3358ade9eacf9bc2bb8ae589863a4f911611694103af05346637df1b7",
+                "sha256:fd67b306320dcadea700a8f79b9e671e607f8696e98ec255915c0c6d6b818503"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==1.25.0"
+            "version": "==1.25.1"
         },
         "odfpy": {
             "hashes": [
@@ -1430,98 +1452,118 @@
         },
         "pydantic": {
             "hashes": [
-                "sha256:041945a6c75f2451a343674ec7d220cb7e207884fb06aaf2c16b6d0bfaf2bc39",
-                "sha256:7a3e3b1d0384eaa313f0810cffa475d6849794a9ae5768989518114771cb9241"
+                "sha256:b802f5245b8576315fe619e5989fd083448fa1258638ef9dac301ca60878396d",
+                "sha256:f5581e0c79b2ec2fa25a9d30d766629811cdda022107fa73d022ab5578873ae3"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.0.1"
+            "version": "==2.0.2"
         },
         "pydantic-core": {
             "hashes": [
-                "sha256:038876cd2dfc1319e0256995ee74cdd90df2ce03bc6060d5eaee01cc78cf3dae",
-                "sha256:07f02b4a474fa89be0bb0b0c42eb605d2a9c8fe11ea7f82fb754060fd0a5ac33",
-                "sha256:0b435c029f00b402df3ab19c07b6d8a2e26a5abbb15117b93c457e3ed40237d7",
-                "sha256:0cec91249c78b5697294b01e66acb819433f4111ae640b7300dd5508a522342e",
-                "sha256:1005ab00b3f39b044408a357b41b66709b6eca17092d2713ee4b79d85a86457b",
-                "sha256:13ff737b9dbda2175bf2d59f8c8d0989b9a331a50d1eb8b7e6e0fdc264af3e93",
-                "sha256:15cb57ca61280eca0b8d721d3629871ab239954c4cec049acf9354405836f341",
-                "sha256:17334cef22055154b7faf7254cc0bf86fea34a7343225b8c6d2d0e54f3533048",
-                "sha256:196c90996542db0151265a1fe7b32d20f5d66fc00ec12ef6f10dd6a3be5aa05f",
-                "sha256:1fa900836d3995ecf34b48f4687a7908b5de85f194e534a7f3a88bfeaee7e25b",
-                "sha256:210ed18f2c438b282a2d5710c07dfa42b8de63647f650c742ecd18a4e02a0618",
-                "sha256:21dcb4f0168f3877cb487dc18362b78bea1e877bcb9c6b4af7563d5e00508cc0",
-                "sha256:24a46c1fd078f3dc7d075200e48b219ed0876f81753201a2d97ad09165d5383f",
-                "sha256:26722063f83c3c4f596adc1eadfa03249afa38e75f3516684de9b57e15d07346",
-                "sha256:26f948f36f679d84cb1b66be40775a09275579e9bba01178dbe9b8231dcbf691",
-                "sha256:27338dfc0a474645d6fe2139b30f006a381f7926e80485370361d7e882a60034",
-                "sha256:27b3eb357a801519dcf42f6c88a3a37e140cf29be21dd5dc152cfc9fa44c34d2",
-                "sha256:2c351a141124c216fe4a0119ef2fa5bc70eec710e59cdd79346475b3f78d15e9",
-                "sha256:2c587db8f31a1c3270991945c20c2ace289fbfa7cf2d533f67f47e95c9ead83e",
-                "sha256:2e02faa4a5e9bd1d7cb4b056c911826f67c4bf298979f89f07c3f2446cd0cf86",
-                "sha256:2f45943b592070fd744660fc8e31a010ae78a6e91f8e6431c07f6dce022eb03f",
-                "sha256:312263dea8116f68972c41c53c0a5b5bf9f7732e7bdc978acb847ed7c9fc8207",
-                "sha256:3127bd2a5764ed08529ca03f8b9e486d347fb2f604cd8333ae7e55a1693073af",
-                "sha256:31f95633f6a3ddc8e0b850157ac0cedb8ccacbe4349310b4be6d724860d8f5c0",
-                "sha256:4277e1941faa5c59fddfc49dae98dc94c16288bc9a09c7b17599c8388aeadcb5",
-                "sha256:433b13fa81a06589dae5198dd285c5621714d4b6d75da058ba8347f8c36cb796",
-                "sha256:4b73e646fda49a5b503f7484a8797a36697b28b5be3adb597460f1d3d337fb82",
-                "sha256:4cd3178131bb7d0d3df947587d76cf9d1ab4318fe45e8ad18dafba3b1f0cda6d",
-                "sha256:4d5cece19558a3490ace346d70322766e670c51ce98ab9bea3e85efba6c00424",
-                "sha256:4ed79de66b4b9acdd613c48befe4afcbee05f6153d793df6922ffc392f46720e",
-                "sha256:51defb4826a28644034915ec5f5a5d3be2d56b683891343d53dfca936c634326",
-                "sha256:5465264bbc535a8650a3806ae5bd07e2691428004a52c961281eadce519c60cc",
-                "sha256:5598f9d4e063e9a64233792dc0f8a0fab8036fb66d25cfc356649667a6542bfb",
-                "sha256:5e931731368ea56f1787fc408757708348639ef2aa1f01e3d483ad1574780b92",
-                "sha256:638b474da73e71079f39a80e4d70196853c2d2fc98c3d425ce3a3ae738e2245f",
-                "sha256:6ccade95f48f47c898632d8dd995704924fce0f99deb7fd4f24348792769abec",
-                "sha256:751e6deca13d89bc5ffc4684ac8a4ea08c6c0ac8dfe12cc5d6927f249879131d",
-                "sha256:75bbf0045f52696aa317b38e67ef5c80a15b7aab572956df2c6fb44f3f4c8b3e",
-                "sha256:7d1c453a36e69ddd4ea47a8e5426a63fdcb731d18122571fbdfda23b07ad28b1",
-                "sha256:7e5264ed7727ab09c410a98c47430c2ab426c2edb9a7b613ca1d785dd3506b7d",
-                "sha256:8136e89efab6f8399bdaf5254758db37049eeaa2f39645ce999aa5162392be28",
-                "sha256:84f1eb4d23a37f77b20dabffe7d5971c6c8eea78bd977fcd2007704ccb540230",
-                "sha256:89d271bae5b6e43936e0365b387d317bd309c5e7c5645b7608b939410fb86968",
-                "sha256:8e63f360661847422423410ebe755258aefad8bd67e9ac516eb1d02a90bdf788",
-                "sha256:8ebb72dec9eefc3eb419de764d0510bbaa08e4db2b4a997576cce338a5f93c97",
-                "sha256:996ffb7ae3c8cb7506a58dae52bbf13a7bbbfce6c3110a2b44c20d2587e57b9b",
-                "sha256:9cf009170f5f93c3dad4c4f73d827541d4bb7099cf69216c091d8cdd33867255",
-                "sha256:9d65b216c0e55414330e46c272896d4858a30d53310aa6e58520e2fc3d122deb",
-                "sha256:a10ce991b6986c91fdf100611d97f76b2950a1d2c2e72be0484565bf95b03767",
-                "sha256:a349f816319ac85759a19ccb0e93992fe77f8e1961a389cd15c3b5c6098bcabd",
-                "sha256:a5576ad07f480a21b38fff2e15d2c90ab3b18f36692065235df237711b402afd",
-                "sha256:aab82425d10bf0624e4a7ac902eed33adae413e827b53d82ae131a10c3130208",
-                "sha256:b024721a940a3311328d50f7cc3d9a7aced0f5ee1fd30c0fa7cbbc542ec3a55c",
-                "sha256:b67ede74b43598feb405a628c83087b3df1066a388ab060cdd5333d061ecf3f5",
-                "sha256:b8b622793e7b7ecb25916f30e91d49424a1f10db08aa151ff7eabd29039ae15c",
-                "sha256:bd9587083b48ec822960a8047249c8119e82749bdf96cecc2e1975322ccb1405",
-                "sha256:bf4bb512eb302acbef4774f65a9ae83edfb283055de7b18b9656b8fda0869652",
-                "sha256:c17fd1d0fef829b364fbbd06aad286b7a73b7b93a46f1967aff1c8f78e5a250a",
-                "sha256:c344dd1c345b2206515edaba0e0bf4aa2b1c456822f3ac9bc0d9f7fc971a8934",
-                "sha256:c44ec0439fac342f773cd848b20cf28cc376670369a6d42845d180f18f2671e3",
-                "sha256:c63bb44c2af1250fcf6e8447b0fda17f09d28e4677910f5bc1328881ae2c527e",
-                "sha256:c7a2c290d6abff5abf6566aa5ea07342e74af42f4defb1f33b3b3d9e7ff1c61f",
-                "sha256:c815a0908065dd8eae0740e55063fcf730c5ef86edf6210ecd53ace3a85c9911",
-                "sha256:c9f856a5c8938f2e0c7bb337f09d5212afd390627929c53e5f0c5944c99732fc",
-                "sha256:ca833856df881b9809747131c38bf7b6af7262ab2c77a2834b9e9d64cf43ab4e",
-                "sha256:d088fdc5cc709a715cf9f49e698a5690cc00616d3379e55d07423e628a21a097",
-                "sha256:d2db12d32b3b83c3d1a2044f9ba31aca9a8224c7eb15d949bdae3e826ee8c6ec",
-                "sha256:d4677520ade160805ad55a6418db7beea9dea34f0a091da1f0bcf09c66091b54",
-                "sha256:dbe2b50a4c3bcc9962449eea1c73d2e509a4e3a96df38511b898eea768fde4a4",
-                "sha256:dc901bb6ffe6d983903242dd7495660161b8901307c5280534fee3b0a90f98e6",
-                "sha256:e28d86253cdc638d084751bcc1217944370c567722d377c1364fd1433d0a41f9",
-                "sha256:e5bcca875379fab98c7b8b4ddfe932844d9ac7dc0a850c5afa414d17988aed93",
-                "sha256:e6973ccb84a532e35b6a9f7f8d6024688186d950278700d408836219aa5b6164",
-                "sha256:e7d6a9e510ae4ea02db709472102fa7b59d48441a6c0419a7d21d0b96672a469",
-                "sha256:e7d8df9e29ecc2930d27fccde99ae86c1dfc42c1f92e81715df2a7dc1f7f466e",
-                "sha256:f295db65d4de14c0b46168a6db73be34b8fe4e3e2699a9c574b37412d0dd2a41",
-                "sha256:f8119485a74487780fecf8c03cce66a2fb13da2e68f4219af7aca9d0eb8ff64d",
-                "sha256:fb3d452def28f86fcec749659fea183650c23aa46ae4d8a9996463a1793587b5",
-                "sha256:fb6551210cef7423d68eaaeab60a9445e17edd33d251b2ab6c783afce9811df8",
-                "sha256:fc4cb821dc67963463f8d8be6dca8933210d050009b32f683d02444a3d5f1e02",
-                "sha256:fede91ea67570eb296d4ae88aecb9c51a46cdccb35a388dba759183ba84c61d6"
+                "sha256:017700236ea2e7afbef5d3803559c80bd8720306778ebd49268de7ce9972e83e",
+                "sha256:047e782b9918f35ef534ced36f1fd2064f5581229b7a15e4d3177387a6b53134",
+                "sha256:0681472245ef182554208a25d16884c84f1c5a69f14e6169b88932e5da739a1c",
+                "sha256:06ae67547251135a1b3f8dd465797b13146295a3866bc12ddd73f7512787bb7c",
+                "sha256:080a7af828388284a68ad7d3d3eac3bcfff6a580292849aff087e7d556ec42d4",
+                "sha256:0855cf8b760fb40f97f0226cb527c8a94a2ab9d8179628beae20d6939aaeacb0",
+                "sha256:087ddbb754575618a8832ee4ab52fe7eb332f502e2a56088b53dbeb5c4efdf9f",
+                "sha256:0b5d37aedea5963f2097bddbcdb255483191646a52d40d8bb66d61c190fcac91",
+                "sha256:0be2e2812a43205728a06c9d0fd090432cd76a9bb5bff2bfcfdf8b0e27d51851",
+                "sha256:0e5761ce986ec709897b1b965fad9743f301500434bea3cbab2b6e662571580f",
+                "sha256:15eb4cb543ed36f6a4f16e3bee7aa7ed1c3757be95a3f3bbb2b82b9887131e0f",
+                "sha256:1635a37137fafbc6ee0a8c879857e05b30b1aabaa927e653872b71f1501b1502",
+                "sha256:1a5c4475510d1a9cc1458a26cfc21442223e52ce9adb640775c38739315d03c7",
+                "sha256:1c917f7a41d9d09b8b024a5d65cf37e5588ccdb6e610d2df565fb7186b1f3b1c",
+                "sha256:2278ca0b0dfbcfb1e12fa58570916dc260dc72bee5e6e342debf5329d8204688",
+                "sha256:24c3c9180a2d19d640bacc2d00f497a9a1f2abadb2a9ee201b56bb03bc5343bd",
+                "sha256:2575664f0a559a7b951a518f6f34c23cab7190f34f8220b8c8218c4f403147ee",
+                "sha256:2ca2d2d5ab65fb40dd05259965006edcc62a9d9b30102737c0a6f45bcbd254e8",
+                "sha256:2ee3ae58f271851362f6c9b33e4c9f9e866557ec7d8c03dc091e9b5aa5566cec",
+                "sha256:3747a4178139ebf3f19541285b2eb7c886890ca4eb7eec851578c02a13cc1385",
+                "sha256:4663293a36a851a860b1299c50837914269fca127434911297dd39fea9667a01",
+                "sha256:46cd323371aa7e4053010ccdb94063a4273aa9e5dbe97f8a1147faa769de8d8d",
+                "sha256:4938b32c09dbcecbeb652327cb4a449b1ef1a1bf6c8fc2c8241aa6b8f6d63b54",
+                "sha256:4ac140d54da366672f6b91f9a1e8e2d4e7e72720143353501ae886d3fca03272",
+                "sha256:4aff436c23c68449601b3fba7075b4f37ef8fbb893c8c1ed3ef898f090332b1e",
+                "sha256:4e67f9b9dfda2e42b39459cbf99d319ccb90da151e35cead3521975b2afbf673",
+                "sha256:5056afea59651c4e47ec6dadbb77ccae4742c059a3d12bc1c0e393d189d2970d",
+                "sha256:51968887d6bd1eaa7fc7759701ea8ccb470c04654beaa8ede6835b0533f206a9",
+                "sha256:5948af62f323252d56acaec8ebfca5f15933f6b72f8dbe3bf21ee97b2d10e3f0",
+                "sha256:5a014ee88980013d192a718cbb88e8cea20acd3afad69bc6d15672d05a49cdb6",
+                "sha256:60b7239206a2f61ad89c7518adfacb3ccd6662eaa07c5e437317aea2615a1f18",
+                "sha256:682ff9228c838018c47dfa89b3d84cca45f88cacde28807ab8296ec221862af4",
+                "sha256:68a2a767953c707d9575dcf14d8edee7930527ee0141a8bb612c22d1f1059f9a",
+                "sha256:6bf00f56a4468f5b03dadb672a5f1d24aea303d4ccffe8a0f548c9e36017edd3",
+                "sha256:6e3bcb4a9bc209a61ea2aceb7433ce2ece32c7e670b0c06848bf870c9b3e7d87",
+                "sha256:7345b1741bf66a9d8ed0ec291c3eabd534444e139e1ea6db5742ac9fd3be2530",
+                "sha256:74a33aa69d476773230396396afb8e11908f8dafdcfd422e746770599a3f889d",
+                "sha256:7648e48ba263ca0a8a2dc55a60a219c9133fb101ba52c89a14a29fb3d4322ca3",
+                "sha256:7684b5fb906b37e940c5df3f57118f32e033af5e4770e5ae2ae56fbd2fe1a30a",
+                "sha256:76c9c55462740d728b344e3a087775846516c3fee31ec56e2075faa7cfcafcbf",
+                "sha256:7c7ad8958aadfbcd664078002246796ecd5566b64b22f6af4fd1bbcec6bf8f60",
+                "sha256:7ff6bfe63f447a509ed4d368a7f4ba6a7abc03bc4744fc3fb30f2ffab73f3821",
+                "sha256:804cf8f6a859620f8eb754c02f7770f61c3e9c519f8338c331d555b3d6976e3c",
+                "sha256:8125152b03dd91deca5afe5b933a1994b39405adf6be2fe8dce3632319283f85",
+                "sha256:817681d111cb65f07d46496eafec815f48e1aff37713b73135a0a9eb4d3610ab",
+                "sha256:818f5cb1b209ab1295087c45717178f4bbbd2bd7eda421f7a119e7b9b736a3cb",
+                "sha256:82e09f27edab289187dd924d4d93f2a35f21aa969699b2504aa643da7fbfeff9",
+                "sha256:840238c845b0f80777151fef0003088ab91c6f7b3467edaff4932b425c4e3c3f",
+                "sha256:87cff210af3258ca0c829e3ebc849d7981bfde23a99d6cb7a3c17a163b3dbad2",
+                "sha256:8884a1dbfc5cb8c54b48446ca916d4577c1f4d901126091e4ab25d00194e065f",
+                "sha256:88a56f0f6d020b4d17641f4b4d1f9540a536d4146768d059c430e97bdb485fc1",
+                "sha256:8b9a5fc4058d64c9c826684dcdb43891c1b474a4a88dcf8dfc3e1fb5889496f8",
+                "sha256:8c0213891898fa5b404cf3edf4797e3ac7819a0708ea5473fc6432a2aa27c189",
+                "sha256:8e6ce261ccb9a986953c4dce070327e4954f9dd4cd214746dfc70efbc713b6a1",
+                "sha256:8eb4e2b71562375609c66a79f89acd4fe95c5cba23473d04952c8b14b6f908f5",
+                "sha256:90b06bb47e60173d24c7cb79670aa8dd6081797290353b9d3c66d3a23e88eb34",
+                "sha256:94d368af9e6563de6e7170a74710a2cbace7a1e9c8e507d9e3ac34c7065d7ae3",
+                "sha256:9a5fba9168fc27805553760fa8198db46eef83bf52b4e87ebbe1333b823d0e70",
+                "sha256:9b9f8bf1d7008a58fbb6eb334dc6e2f2905400cced8dadb46c4ca28f005a8562",
+                "sha256:a4ae46769d9a7138d58cd190441cac14ce954010a0081f28462ed916c8e55a4f",
+                "sha256:a772c652603855d7180015849d483a1f539351a263bb9b81bfe85193a33ce124",
+                "sha256:a8b9c2cc4c5f8169b943d24be4bd1548fe81c016d704126e3a3124a2fc164885",
+                "sha256:aa39499625239da4ec960cf4fc66b023929b24cc77fb8520289cfdb3c1986428",
+                "sha256:aa54902fa51f7d921ba80923cf1c7ff3dce796a7903300bd8824deb90e357744",
+                "sha256:ac462a28218ea7d592c7ad51b517558f4ac6565a4e53db7a4811eeaf9c9660b0",
+                "sha256:af832edd384755826e494ffdcf1fdda86e4babc42a0b18d342943fb18181040e",
+                "sha256:b1fad38db1744d27061df516e59c5025b09b0a50a337c04e6eebdbddc18951bc",
+                "sha256:b4038869ba1d8fa33863b4b1286ab07e6075a641ae269b865f94d7e10b3e800e",
+                "sha256:b4673d1f29487608d613ebcc5caa99ba15eb58450a7449fb6d800f29d90bebc1",
+                "sha256:b4815720c266e832b20e27a7a5f3772bb09fdedb31a9a34bab7b49d98967ef5a",
+                "sha256:b59a64c367f350873c40a126ffe9184d903d2126c701380b4b55753484df5948",
+                "sha256:b74906e01c7fc938ac889588ef438de812989817095c3c4904721f647d64a4d1",
+                "sha256:b815a769b019dd96be6571096f246b74f63330547e9b30244c51b4a2eb0277fc",
+                "sha256:bad7029fb2251c1ac7d3acdd607e540d40d137a7d43a5e5acdcfdbd38db3fc0a",
+                "sha256:bb471ea8650796060afc99909d9b75da583d317e52f660faf64c45f70b3bf1e2",
+                "sha256:bd95d223de5162811a7b36c73d48eac4fee03b075132f3a1b73c132ce157a60c",
+                "sha256:be3419204952bbe9b72b90008977379c52f99ae1c6e640488de4be783c345d71",
+                "sha256:c0f481aaf0119f77b200e5a5e2799b3e14c015a317eaa948f42263908735cc9f",
+                "sha256:c2d00a96fdf26295c6f25eaf9e4a233f353146a73713cd97a5f5dc6090c3aef2",
+                "sha256:c720e55cef609d50418bdfdfb5c44a76efc020ae7455505788d0113c54c7df55",
+                "sha256:cb854ec52e6e2e05b83d647695f4d913452fdd45a3dfa8233d7dab5967b3908f",
+                "sha256:cbba32fb14e199d0493c6b9c44870dab0a9c37af9f0f729068459d1849279ffd",
+                "sha256:cd62f73830d4715bc643ae39de0bd4fb9c81d6d743530074da91e77a2cccfe67",
+                "sha256:cf92dccca8f66e987f6c4378700447f82b79e86407912ab1ee06b16b82f05120",
+                "sha256:d281a10837d98db997c0247f45d138522c91ce30cf3ae7a6afdb5e709707d360",
+                "sha256:d2c790f0d928b672484eac4f5696dd0b78f3d6d148a641ea196eb49c0875e30a",
+                "sha256:d35d634d9d1ed280c87bc2a7a6217b8787eedc86f368fc2fa1c0c8c78f7d3c93",
+                "sha256:db4564aea8b3cb6cf1e5f3fd80f1ced73a255d492396d1bd8abd688795b34d63",
+                "sha256:dc737506b4a0ba2922a2626fc6d620ce50a46aebd0fe2fbcad1b93bbdd8c7e78",
+                "sha256:e17056390068afd4583d88dcf4d4495764e4e2c7d756464468e0d21abcb8931e",
+                "sha256:e68a404fad8493989d6f07b7b9e066f1d2524d7cb64db2d4e9a84c920032c67f",
+                "sha256:e7fd334b40c5e13a97becfcaba314de0dcc6f7fe21ec8f992139bcc64700e9dc",
+                "sha256:eb4301f009a44bb5db5edfe4e51a8175a4112b566baec07f4af8b1f8cb4649a2",
+                "sha256:ebf583f4d9b52abd15cc59e5f6eeca7e3e9741c6ea62d8711c00ac3acb067875",
+                "sha256:ef71e73a81a4cd7e87c93e8ff0170140fd93ba33b0f61e83da3f55f6e0a84fb4",
+                "sha256:f0eb54b11cd4fe0c6404611eef77086ade03eb1457e92910bbb4f3479efa3f79",
+                "sha256:f2de65752fff248319bcd3b29da24e205fa505607539fcd4acc4037355175b63",
+                "sha256:f5de2d4167fd4bc5ad205fb7297e25867b8e335ca08d64ed7a561d2955a2c32d",
+                "sha256:f7bcdf70c8b6e70be11c78d3c00b80a24cccfb408128f23e91ec3019bed1ecc1",
+                "sha256:fa38a76e832743866aed6b715869757074b06357d1a260163ec26d84974245fe",
+                "sha256:fc909f62325a631e1401dd07dfc386986dbcac15f98c9ff2145d930678a9d25a"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.0.2"
+            "version": "==2.1.2"
         },
         "pymysql": {
             "hashes": [
@@ -1706,6 +1748,31 @@
             "markers": "python_version >= '3.7'",
             "version": "==0.6.1"
         },
+        "scipy": {
+            "hashes": [
+                "sha256:08d957ca82d3535b3b9ba6c8ff355d78fe975271874e2af267cb5add5bd78625",
+                "sha256:249cfa465c379c9bb2c20123001e151ff5e29b351cbb7f9c91587260602c58d0",
+                "sha256:366a6a937110d80dca4f63b3f5b00cc89d36f678b2d124a01067b154e692bab1",
+                "sha256:39154437654260a52871dfde852adf1b93b1d1bc5dc0ffa70068f16ec0be2624",
+                "sha256:396fae3f8c12ad14c5f3eb40499fd06a6fef8393a6baa352a652ecd51e74e029",
+                "sha256:3b9963798df1d8a52db41a6fc0e6fa65b1c60e85d73da27ae8bb754de4792481",
+                "sha256:3e8eb42db36526b130dfbc417609498a6192381abc1975b91e3eb238e0b41c1a",
+                "sha256:512fdc18c65f76dadaca139348e525646d440220d8d05f6d21965b8d4466bccd",
+                "sha256:aec8c62fbe52914f9cf28d846cf0401dd80ab80788bbab909434eb336ed07c04",
+                "sha256:b41a0f322b4eb51b078cb3441e950ad661ede490c3aca66edef66f4b37ab1877",
+                "sha256:b4bb943010203465ac81efa392e4645265077b4d9e99b66cf3ed33ae12254173",
+                "sha256:b588311875c58d1acd4ef17c983b9f1ab5391755a47c3d70b6bd503a45bfaf71",
+                "sha256:ba94eeef3c9caa4cea7b402a35bb02a5714ee1ee77eb98aca1eed4543beb0f4c",
+                "sha256:be8c962a821957fdde8c4044efdab7a140c13294997a407eaee777acf63cbf0c",
+                "sha256:cce154372f0ebe88556ed06d7b196e9c2e0c13080ecb58d0f35062dc7cc28b47",
+                "sha256:d51565560565a0307ed06fa0ec4c6f21ff094947d4844d6068ed04400c72d0c3",
+                "sha256:e866514bc2d660608447b6ba95c8900d591f2865c07cca0aa4f7ff3c4ca70f30",
+                "sha256:fb5b492fa035334fd249f0973cc79ecad8b09c604b42a127a677b45a9a3d4289",
+                "sha256:ffb28e3fa31b9c376d0fb1f74c1f13911c8c154a760312fbee87a21eb21efe31"
+            ],
+            "markers": "python_version < '3.13' and python_version >= '3.9'",
+            "version": "==1.11.1"
+        },
         "setuptools": {
             "hashes": [
                 "sha256:d0b9a8433464d5800cbe05094acf5c6d52a91bfac9b52bcfc4d41382be5d5d31",
@@ -1774,11 +1841,11 @@
         },
         "types-awscrt": {
             "hashes": [
-                "sha256:25ed116be03203fddd889bb46db2923f655490c9e7fe4734176ebef53c0e782a",
-                "sha256:c7ec33cf74734f79bfcf0f9a76aaa7feac40c34f457a3de7977be1d8fdf02a37"
+                "sha256:5a846b77a6d35b63ccf9d33abf386c4dfc6275f9c062bc0e234c14ea8c6013a9",
+                "sha256:7edec4283fec9f92574444ce45fc0be65c846056634628aee22c118712da71c4"
             ],
             "markers": "python_version >= '3.7' and python_version < '4.0'",
-            "version": "==0.16.21"
+            "version": "==0.16.23"
         },
         "types-pymysql": {
             "hashes": [

From 75c3c6dd60e908a20fef0c33ad2eed23ccaf5366 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Mon, 10 Jul 2023 17:00:29 +0800
Subject: [PATCH 085/161] feat: update patch

---
 dongtai_conf/patch/__init__.py | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/dongtai_conf/patch/__init__.py b/dongtai_conf/patch/__init__.py
index 2149dd169..8681aef7d 100644
--- a/dongtai_conf/patch/__init__.py
+++ b/dongtai_conf/patch/__init__.py
@@ -1,8 +1,13 @@
+import importlib
 import inspect
 import logging
+import pkgutil
 from dataclasses import dataclass
+from pathlib import Path
 from types import CodeType
-from typing import Callable
+from typing import Any, Callable
+
+from dongtai_conf.settings import BASE_DIR
 
 logger = logging.getLogger("patch")
 
@@ -12,10 +17,23 @@ class PatchConfig:
     type_check: bool
 
 
+is_init_patch = False
 PATCH_HANDLER: dict[CodeType, tuple[Callable, PatchConfig]] = {}
 
 
-def patch_point(*args) -> None:
+def init_patch() -> None:
+    global is_init_patch
+    if not is_init_patch:
+        PATCH_ROOT_PATH = Path(BASE_DIR) / "dongtai_conf" / "patch"
+        for module_info in pkgutil.iter_modules([str(PATCH_ROOT_PATH.resolve())]):
+            if not module_info.name.startswith("_"):
+                importlib.import_module("dongtai_conf.patch." + module_info.name)
+        is_init_patch = True
+    print(PATCH_HANDLER)
+
+
+def patch_point(*args) -> Any | None:
+    init_patch()
     current_frame = inspect.currentframe()
     if current_frame is None:
         logger.error("current frame is None, can not patch")
@@ -60,7 +78,7 @@ def patch_point(*args) -> None:
                 logger.error(f"can not call patch function, miss local var {name}")
                 return
             count += 1
-        func(**patch_func_args)
+        return func(**patch_func_args)
 
 
 def patch(patch_func: Callable, type_check: bool = False):
@@ -71,7 +89,7 @@ def wrapper(func: Callable):
     return wrapper
 
 
-def check_patch():
+def check_patch() -> None:
     for code, func in PATCH_HANDLER.items():
         args, _, _, _, kwonlyargs, _, _ = inspect.getfullargspec(func)
         if not set(args + kwonlyargs).issubset(set(code.co_varnames)):

From e0d0fe45a6bfc89b8c80d91f422c6070f6c7967b Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Mon, 10 Jul 2023 17:04:45 +0800
Subject: [PATCH 086/161] fix: max recursive error.

---
 requirements.txt | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index d6e179c3a..d03531626 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,7 +10,7 @@ attrs==23.1.0 ; python_version >= '3.7'
 autopep8==2.0.2 ; python_version >= '3.6'
 billiard==4.1.0 ; python_version >= '3.7'
 boto3==1.24.59
-boto3-stubs==1.27.0
+boto3-stubs==1.28.1
 botocore==1.27.91
 botocore-stubs==1.29.165
 celery==5.3.0rc1
@@ -18,8 +18,8 @@ celery-singleton==0.3.1
 certifi==2023.5.7
 cffi==1.15.1
 chardet==5.1.0
-charset-normalizer==3.1.0 ; python_full_version >= '3.7.0'
-click==8.1.3 ; python_version >= '3.7'
+charset-normalizer==3.2.0 ; python_full_version >= '3.7.0'
+click==8.1.4 ; python_version >= '3.7'
 click-didyoumean==0.3.0 ; python_full_version >= '3.6.2' and python_full_version < '4.0.0'
 click-plugins==1.1.1
 click-repl==0.3.0 ; python_version >= '3.6'
@@ -79,11 +79,12 @@ markupsafe==2.1.3 ; python_version >= '3.7'
 marshmallow==3.19.0 ; python_version >= '3.7'
 marshmallow-enum==1.5.1
 mock==5.0.2 ; python_version >= '3.6'
-model-bakery==1.12.0 ; python_version >= '3'
+model-bakery==1.12.0 ; python_version >= '3.7'
 mypy==1.0.1
 mypy-extensions==1.0.0 ; python_version >= '3.5'
 mysqlclient==2.2.0
-numpy==1.25.0 ; python_version >= '3.9'
+networkit==10.1
+numpy==1.25.1 ; python_version >= '3.9'
 odfpy==1.4.1
 openpyxl==3.0.9
 oss2==2.13.1
@@ -95,8 +96,8 @@ pycodestyle==2.10.0 ; python_version >= '3.6'
 pycparser==2.21
 pycryptodome==3.18.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 pycryptodomex==3.14.1
-pydantic==2.0.1 ; python_version >= '3.7'
-pydantic-core==2.0.2 ; python_version >= '3.7'
+pydantic==2.0.2 ; python_version >= '3.7'
+pydantic-core==2.1.2 ; python_version >= '3.7'
 pymysql==1.0.2
 pyparsing==3.1.0 ; python_full_version >= '3.6.8'
 pyre2==0.3.6
@@ -111,6 +112,7 @@ redis==4.4.4
 requests==2.31.0
 result==0.8.0
 s3transfer==0.6.1 ; python_version >= '3.7'
+scipy==1.11.1 ; python_version < '3.13' and python_version >= '3.9'
 setuptools==65.5.1
 shortuuid==1.0.11
 simhash==2.1.2
@@ -118,7 +120,7 @@ six==1.15.0
 sqlparse==0.4.4 ; python_version >= '3.5'
 tablib[html,ods,xls,xlsx,yaml]==3.5.0 ; python_version >= '3.8'
 tomli==2.0.1 ; python_version < '3.11'
-types-awscrt==0.16.21 ; python_version >= '3.7' and python_version < '4.0'
+types-awscrt==0.16.23 ; python_version >= '3.7' and python_version < '4.0'
 types-pymysql==1.1.0.0
 types-pyopenssl==23.2.0.1
 types-python-dateutil==2.8.19.13

From 21efbd652a821d279b66ed91f8b4cb1702c91aac Mon Sep 17 00:00:00 2001
From: st1020 
Date: Mon, 10 Jul 2023 17:42:59 +0800
Subject: [PATCH 087/161] fix: project add error

---
 dongtai_web/views/project_add.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dongtai_web/views/project_add.py b/dongtai_web/views/project_add.py
index f2c8cb9d5..497bf98f0 100644
--- a/dongtai_web/views/project_add.py
+++ b/dongtai_web/views/project_add.py
@@ -77,7 +77,7 @@ def post(self, request):
             with transaction.atomic():
                 name = request.data.get("name")
                 mode = "插桩模式"
-                scan_id = int(request.data.get("scan_id", 0))
+                scan_id = int(request.data.get("scan_id", 5))
                 template_id = int(request.data.get("template_id", 0))
                 # auth_users = self.get_auth_users(request.user)
                 departments = request.user.get_relative_department()

From bd327c3fb8adef420c88c3c908881e84cafbf048 Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Mon, 10 Jul 2023 17:54:08 +0800
Subject: [PATCH 088/161] fix: validator not working in validate command and
 tag.

---
 dongtai_common/common/agent_command_check.py | 17 +++++++++++++++--
 dongtai_conf/settings.py                     |  2 +-
 dongtai_web/views/engine_hook_rule_add.py    | 11 ++++++-----
 dongtai_web/views/engine_hook_rule_modify.py |  8 +++++---
 4 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/dongtai_common/common/agent_command_check.py b/dongtai_common/common/agent_command_check.py
index 918285b32..ef7cf4f86 100644
--- a/dongtai_common/common/agent_command_check.py
+++ b/dongtai_common/common/agent_command_check.py
@@ -1,9 +1,12 @@
 import re
 from dongtai_conf.settings import DEFAULT_TAINT_VALUE_RANGE_COMMANDS, DEFAULT_IAST_VALUE_TAG
+from rest_framework.serializers import ValidationError
+
+PATTERN = rf" *({'|'.join(DEFAULT_TAINT_VALUE_RANGE_COMMANDS)}) *\(( *(P\d+|\d+) *,*)*\) *"
+
 
 def valitate_taint_command(command: str) -> bool:
-    pattern = rf" *({'|'.join(DEFAULT_TAINT_VALUE_RANGE_COMMANDS)}) *\(( *(P\d+|\d+) *,*)*\) *"
-    res = re.fullmatch(pattern, command.upper(), flags=re.IGNORECASE)
+    res = re.fullmatch(PATTERN, command.upper(), flags=re.IGNORECASE)
     if res:
         return True
     return False
@@ -13,3 +16,13 @@ def valitate_tag(tag: str) -> bool:
     if tag not in DEFAULT_IAST_VALUE_TAG:
         return False
     return True
+
+
+def taint_command_validator(command: str):
+    if not valitate_taint_command(command):
+        raise ValidationError(f'The command must fit {PATTERN} .')
+
+
+def tag_validator(tag: str):
+    if not valitate_tag(tag):
+        raise ValidationError(f'The tag must in {DEFAULT_IAST_VALUE_TAG} .')
diff --git a/dongtai_conf/settings.py b/dongtai_conf/settings.py
index 2cfc7fe65..e308c4e76 100644
--- a/dongtai_conf/settings.py
+++ b/dongtai_conf/settings.py
@@ -684,7 +684,7 @@ def safe_execute(default, exception, function, *args):
     'xpath-decoded', 'ldap-encoded', 'ldap-decoded',
     'http-token-limited-chars', 'numeric-limited-chars'
 ]
-DEFAULT_TAINT_VALUE_RANGE_COMMANDS = ['KEEP', 'APPEND', 'SUBSET', 'INSERT', 'REMOVE', 'REPLACE', 'CONCAT', 'TRIM', 'TRIM_RIGHT', 'TRIM_LEFT']
+DEFAULT_TAINT_VALUE_RANGE_COMMANDS = ['KEEP', 'APPEND', 'SUBSET', 'INSERT', 'REMOVE', 'REPLACE', 'CONCAT', 'TRIM', 'TRIM_RIGHT', 'TRIM_LEFT', 'OVERWRITE']
 DEFAULT_CODE_DETECT_BLACK_LIST = [
     'aj.', 'akka.', 'android.', 'antlr.', 'apple.', 'aQute.', 'brave.', 'bsh.',
     'ch.qos.', 'co.paralleluniverse.', 'com.acumenat.', 'com.alibaba.arthas.',
diff --git a/dongtai_web/views/engine_hook_rule_add.py b/dongtai_web/views/engine_hook_rule_add.py
index 60daf7a52..9b2f7aadb 100644
--- a/dongtai_web/views/engine_hook_rule_add.py
+++ b/dongtai_web/views/engine_hook_rule_add.py
@@ -18,8 +18,8 @@
 from rest_framework.serializers import ValidationError
 from dongtai_common.models.strategy import IastStrategyModel
 from dongtai_common.common.agent_command_check import (
-    valitate_taint_command,
-    valitate_tag,
+    tag_validator,
+    taint_command_validator,
 )
 logger = logging.getLogger('dongtai-webapi')
 
@@ -62,18 +62,18 @@ class _HookRuleAddBodyargsSerializer(serializers.Serializer):
         default=False,
     )
     tags = serializers.ListField(
-        child=serializers.CharField(validators=[valitate_tag]),
+        child=serializers.CharField(validators=[tag_validator]),
         required=False,
         default=list,
     )
     untags = serializers.ListField(
-        child=serializers.CharField(validators=[valitate_tag]),
+        child=serializers.CharField(validators=[tag_validator]),
         required=False,
         default=list,
     )
     command = serializers.CharField(
         max_length=255,
-        validators=[valitate_taint_command],
+        validators=[taint_command_validator],
         required=False,
         default="",
         allow_blank=True,
@@ -189,6 +189,7 @@ def post(self, request):
         except ValidationError as e:
             return R.failure(data=e.detail,
                              msg=_('Incomplete parameter, please check again'))
+        import pdb;pdb.set_trace()
         if rule_target == "":
             hook_type = IastStrategyModel.objects.filter(
                 id=rule_type,
diff --git a/dongtai_web/views/engine_hook_rule_modify.py b/dongtai_web/views/engine_hook_rule_modify.py
index 9d7d88060..4007542a3 100644
--- a/dongtai_web/views/engine_hook_rule_modify.py
+++ b/dongtai_web/views/engine_hook_rule_modify.py
@@ -19,6 +19,8 @@
 from dongtai_common.common.agent_command_check import (
     valitate_taint_command,
     valitate_tag,
+    tag_validator,
+    taint_command_validator,
 )
 
 _PostResponseSerializer = get_response_serializer(status_msg_keypair=(
@@ -68,18 +70,18 @@ class _EngineHookRuleModifySerializer(serializers.Serializer):
         default=False,
     )
     tags = serializers.ListField(
-        child=serializers.CharField(validators=[valitate_tag]),
+        child=serializers.CharField(validators=[tag_validator]),
         required=False,
         default=list,
     )
     untags = serializers.ListField(
-        child=serializers.CharField(validators=[valitate_tag]),
+        child=serializers.CharField(validators=[tag_validator]),
         required=False,
         default=list,
     )
     command = serializers.CharField(
         max_length=256,
-        validators=[valitate_taint_command],
+        validators=[taint_command_validator],
         required=False,
         default="",
         allow_blank=True,

From 5097a819dd5e0ed96763a69896a336cf48bcaa6d Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Mon, 10 Jul 2023 17:54:56 +0800
Subject: [PATCH 089/161] fix: validator not working in validate command and
 tag.

---
 dongtai_web/views/engine_hook_rule_modify.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/dongtai_web/views/engine_hook_rule_modify.py b/dongtai_web/views/engine_hook_rule_modify.py
index 4007542a3..031c7844c 100644
--- a/dongtai_web/views/engine_hook_rule_modify.py
+++ b/dongtai_web/views/engine_hook_rule_modify.py
@@ -17,8 +17,6 @@
 from dongtai_common.models.hook_type import HookType
 from dongtai_common.models.strategy import IastStrategyModel
 from dongtai_common.common.agent_command_check import (
-    valitate_taint_command,
-    valitate_tag,
     tag_validator,
     taint_command_validator,
 )

From 6c3125aaaaedb8af27a91e0dece7fd419d0ce896 Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Mon, 10 Jul 2023 18:00:03 +0800
Subject: [PATCH 090/161] fix: validator not working in validate command and
 tag.

---
 dongtai_web/views/engine_hook_rule_add.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/dongtai_web/views/engine_hook_rule_add.py b/dongtai_web/views/engine_hook_rule_add.py
index 9b2f7aadb..b6a13d96b 100644
--- a/dongtai_web/views/engine_hook_rule_add.py
+++ b/dongtai_web/views/engine_hook_rule_add.py
@@ -189,7 +189,6 @@ def post(self, request):
         except ValidationError as e:
             return R.failure(data=e.detail,
                              msg=_('Incomplete parameter, please check again'))
-        import pdb;pdb.set_trace()
         if rule_target == "":
             hook_type = IastStrategyModel.objects.filter(
                 id=rule_type,

From 0bd8c0fc45a361f19da1441f6f06498c7a4c16b0 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Mon, 10 Jul 2023 18:19:24 +0800
Subject: [PATCH 091/161] fix: project add error

---
 dongtai_web/views/project_add.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dongtai_web/views/project_add.py b/dongtai_web/views/project_add.py
index 497bf98f0..c29f2d920 100644
--- a/dongtai_web/views/project_add.py
+++ b/dongtai_web/views/project_add.py
@@ -122,7 +122,7 @@ def post(self, request):
                         id=pid, department__in=departments).first()
                     project.name = name
                 else:
-                    department_id = request.data.get("department_id")
+                    department_id = request.data.get("department_id", 1)
                     if not departments.filter(pk=department_id).exists():
                         return R.failure(
                             status=203,

From f67181d8d80d4d6bc59aa89a946ac3e852369446 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Mon, 10 Jul 2023 18:26:50 +0800
Subject: [PATCH 092/161] fix: project add error

---
 dongtai_web/views/project_add.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dongtai_web/views/project_add.py b/dongtai_web/views/project_add.py
index c29f2d920..031503ac2 100644
--- a/dongtai_web/views/project_add.py
+++ b/dongtai_web/views/project_add.py
@@ -78,7 +78,7 @@ def post(self, request):
                 name = request.data.get("name")
                 mode = "插桩模式"
                 scan_id = int(request.data.get("scan_id", 5))
-                template_id = int(request.data.get("template_id", 0))
+                template_id = int(request.data.get("template_id", 1))
                 # auth_users = self.get_auth_users(request.user)
                 departments = request.user.get_relative_department()
                 scan = IastStrategyUser.objects.filter(id=scan_id).first()

From 2313a135d03bbd6214ecf3206538f02503213ae5 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Mon, 10 Jul 2023 18:32:24 +0800
Subject: [PATCH 093/161] feat: update patch

---
 dongtai_conf/patch/__init__.py | 66 +++++++++++++++++++++++-----------
 1 file changed, 46 insertions(+), 20 deletions(-)

diff --git a/dongtai_conf/patch/__init__.py b/dongtai_conf/patch/__init__.py
index 8681aef7d..5ff724c14 100644
--- a/dongtai_conf/patch/__init__.py
+++ b/dongtai_conf/patch/__init__.py
@@ -5,7 +5,9 @@
 from dataclasses import dataclass
 from pathlib import Path
 from types import CodeType
-from typing import Any, Callable
+from typing import Any, Callable, TypeVar, overload
+
+from typing_extensions import TypeVarTuple, Unpack
 
 from dongtai_conf.settings import BASE_DIR
 
@@ -32,39 +34,40 @@ def init_patch() -> None:
     print(PATCH_HANDLER)
 
 
-def patch_point(*args) -> Any | None:
+T = TypeVar("T")
+Ts = TypeVarTuple("Ts")
+
+
+@overload
+def patch_point(*args: Unpack[tuple[T]]) -> T:
+    ...
+
+
+@overload
+def patch_point(*args: Unpack[Ts]) -> tuple[Unpack[Ts]]:
+    ...
+
+
+def patch_point(*args: Any) -> Any:
     init_patch()
     current_frame = inspect.currentframe()
     if current_frame is None:
         logger.error("current frame is None, can not patch")
-        return
+        return _return_args(*args)
     caller_frame = current_frame.f_back
     if caller_frame is None:
         logger.error("caller frame is None, can not patch")
-        return
+        return _return_args(*args)
     caller_code = caller_frame.f_code
     if caller_code in PATCH_HANDLER:
         func, patch_config = PATCH_HANDLER[caller_code]
         func_args, _, _, _, kwonlyargs, _, annotations = inspect.getfullargspec(func)
-
         func_args += kwonlyargs
 
-        if args and len(args) != len(func_args):
-            # 如果显式传入参数，进行参数数量检查
-            logger.error(f"args number error, expect {len(func_args)}, get {len(args)}")
-            return
-
         patch_func_args = {}
-        count = 0
         for name in func_args:
             if name in caller_frame.f_locals:
                 local_value = caller_frame.f_locals[name]
-                if args and args[count] is not local_value:
-                    # 如果显式传入参数，进行参数检查
-                    logger.error(
-                        f"args error, expect arg name {name}, get value {args[count]}"
-                    )
-                    return
                 if patch_config.type_check:
                     # 如果启用类型检查，进行类型检查
                     type_ = annotations.get(name, None)
@@ -76,9 +79,32 @@ def patch_point(*args) -> Any | None:
                 patch_func_args[name] = local_value
             else:
                 logger.error(f"can not call patch function, miss local var {name}")
-                return
-            count += 1
-        return func(**patch_func_args)
+                return _return_args(*args)
+        return_value = func(**patch_func_args)
+        if return_value is None:
+            return _return_args(*args)
+        elif len(args) == 1:
+            return return_value
+        elif not isinstance(return_value, tuple):
+            logger.error(
+                f"return value type error: expect tuple, get {type(return_value)}"
+            )
+            return _return_args(*args)
+        elif len(return_value) != len(args):
+            logger.error(
+                f"return value len error: expect {len(args)}, get {len(return_value)}"
+            )
+            return _return_args(*args)
+        else:
+            return _return_args(*return_value)
+    return _return_args(*args)
+
+
+def _return_args(*args: Unpack[Ts]) -> tuple[Unpack[Ts]] | Any:
+    print(args)
+    if len(args) == 1:
+        return args[0]
+    return args
 
 
 def patch(patch_func: Callable, type_check: bool = False):

From 1d0053ab74b115061d92967014cf26c6d555758b Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Tue, 11 Jul 2023 10:21:19 +0800
Subject: [PATCH 094/161] feat: project version timestamp update.

---
 dongtai_conf/celery.py                         |  1 +
 dongtai_engine/plugins/project_time_update.py  | 17 +++++++++++++++++
 dongtai_engine/plugins/strategy_headers.py     |  7 ++++++-
 dongtai_engine/signals/handlers/vul_handler.py |  5 ++++-
 4 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/dongtai_conf/celery.py b/dongtai_conf/celery.py
index dbca3ecc9..705d49f36 100644
--- a/dongtai_conf/celery.py
+++ b/dongtai_conf/celery.py
@@ -71,6 +71,7 @@
     "dongtai_protocol.report.handler.api_route_handler.api_route_gather": {'queue': 'dongtai-api-route-handler', 'routing_key': 'dongtai-api-route-handler'},
     "dongtai_engine.tasks.search_vul_from_method_pool": {'queue': 'dongtai-method-pool-scan', 'routing_key': 'dongtai-method-pool-scan'},
     "dongtai_engine.plugins.project_time_update.project_time_stamp_update": {'queue': 'dongtai-project-time-stamp-update', 'routing_key': 'dongtai-project-time-stamp-update'},
+    "dongtai_engine.plugins.project_time_update.project_version_time_stamp_update": {'queue': 'dongtai-project-time-stamp-update', 'routing_key': 'dongtai-project-time-stamp-update'},
     "dongtai_engine.tasks.search_vul_from_replay_method_pool": {'exchange': 'dongtai-replay-vul-scan', 'routing_key': 'dongtai-replay-vul-scan'},
     "dongtai_web.dongtai_sca.scan.utils.update_one_sca": {'exchange': 'dongtai-sca-task', 'routing_key': 'dongtai-sca-task'},
     "dongtai_engine.preheat.function_flush": {'exchange': 'dongtai-function-flush-data', 'routing_key': 'dongtai-function-flush-data'},
diff --git a/dongtai_engine/plugins/project_time_update.py b/dongtai_engine/plugins/project_time_update.py
index 95201ed0d..286bfbfa6 100644
--- a/dongtai_engine/plugins/project_time_update.py
+++ b/dongtai_engine/plugins/project_time_update.py
@@ -2,6 +2,8 @@
 from celery import shared_task
 from time import time
 from dongtai_common.models.project import IastProject
+from dongtai_common.models.project_version import IastProjectVersion
+
 
 @shared_task(
     queue='dongtai-project-time-stamp-update',
@@ -11,3 +13,18 @@
 def project_time_stamp_update(project_id):
     timestamp = int(time())
     IastProject.objects.filter(pk=project_id).update(latest_time=timestamp)
+
+
+@shared_task(
+    queue='dongtai-project-time-stamp-update',
+    base=Singleton,
+    lock_expiry=20,
+)
+def project_version_time_stamp_update(project_version_id):
+    timestamp = int(time())
+    IastProjectVersion.objects.filter(pk=project_version_id).update(
+        update_time=timestamp)
+from dongtai_engine.plugins.project_time_update import (
+    project_time_stamp_update,
+    project_version_time_stamp_update,
+)
diff --git a/dongtai_engine/plugins/strategy_headers.py b/dongtai_engine/plugins/strategy_headers.py
index e73b239b8..11cf1635f 100644
--- a/dongtai_engine/plugins/strategy_headers.py
+++ b/dongtai_engine/plugins/strategy_headers.py
@@ -19,7 +19,10 @@
 from dongtai_web.vul_log.vul_log import log_vul_found
 from dongtai_common.models.header_vulnerablity import IastHeaderVulnerability, IastHeaderVulnerabilityDetail
 from django.db import IntegrityError
-from dongtai_engine.plugins.project_time_update import project_time_stamp_update
+from dongtai_engine.plugins.project_time_update import (
+    project_time_stamp_update,
+    project_version_time_stamp_update,
+)
 
 
 class FakeSocket():
@@ -137,6 +140,8 @@ def save_vul(vul_type, method_pool, position=None, data=None):
     timestamp = int(time.time())
     project_time_stamp_update.apply_async(
         (method_pool.agent.bind_project_id, ), countdown=5)
+    project_version_time_stamp_update.apply_async(
+        (method_pool.agent.project_version_id, ), countdown=5)
     if vul:
         vul.url = ''
         vul.req_header = method_pool.req_header
diff --git a/dongtai_engine/signals/handlers/vul_handler.py b/dongtai_engine/signals/handlers/vul_handler.py
index f90f38476..eeebb2430 100644
--- a/dongtai_engine/signals/handlers/vul_handler.py
+++ b/dongtai_engine/signals/handlers/vul_handler.py
@@ -21,7 +21,10 @@
 from typing import List, Optional, Callable
 from collections import defaultdict
 from dongtai_common.models.profile import IastProfile
-from dongtai_engine.plugins.project_time_update import project_time_stamp_update
+from dongtai_engine.plugins.project_time_update import (
+    project_time_stamp_update,
+    project_version_time_stamp_update,
+)
 
 
 def equals(source, target):

From 69e3c9074682d975eda9b3738c094968c966e309 Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Tue, 11 Jul 2023 10:30:27 +0800
Subject: [PATCH 095/161] fix: reduce logic and add some comments.

---
 dongtai_common/engine/vul_engine.py | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/dongtai_common/engine/vul_engine.py b/dongtai_common/engine/vul_engine.py
index fd11f3240..8e1d9cad7 100644
--- a/dongtai_common/engine/vul_engine.py
+++ b/dongtai_common/engine/vul_engine.py
@@ -146,6 +146,8 @@ def search(self, method_pool, vul_method_signature, vul_type=None):
         vul_methods = list(
             map(lambda x: x['invokeId'],
                 filter(self.hit_vul_method, self.method_pool)))
+        # Ignore `org.springframework.web.util.pattern.PathPattern.getPatternString()` as a non-source method.
+        # It is only to indicate that the API pattern.
         source_methods = list(
             map(
                 lambda x: x['invokeId'],
@@ -153,23 +155,18 @@ def search(self, method_pool, vul_method_signature, vul_type=None):
                     lambda x: x.get('source', False) and x.get('signature') !=
                     'org.springframework.web.util.pattern.PathPattern.getPatternString()',
                     self.method_pool)))
-        # build a graph
+        # Build a graph
         g = nk.Graph(weighted=True, directed=True)
         for pool in self.method_pool:
-            hashs = list(pool['sourceHash'])
-            vecs = list(
-                product(
-                    [pool['invokeId']],
-                    reduce(lambda x, y: x | y,
-                           [source_hash_dict[i] for i in pool['targetHash']],
-                           set()),
-                ))
+            vecs = [[pool['invokeId'], t] for t in reduce(
+                lambda x, y: x | y,
+                [source_hash_dict[i] for i in pool['targetHash']], set())]
             for source, target in vecs:
                 g.addEdge(source,
                           target,
                           abs(source - target)**1.1,
                           addMissing=True)
-        # checkout each pair source/target have a path or not
+        # Checkout each pair source/target have a path or not
         # It may lost sth when muliti paths exists.
         for s, t in product(source_methods, vul_methods):
             dij_obj = nk.distance.BidirectionalDijkstra(g, s, t).run()

From e273d7ce04119383752258bc15cf6a08b2538082 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Tue, 11 Jul 2023 11:01:33 +0800
Subject: [PATCH 096/161] feat: update patch

---
 dongtai_conf/patch/__init__.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/dongtai_conf/patch/__init__.py b/dongtai_conf/patch/__init__.py
index 5ff724c14..9f833c234 100644
--- a/dongtai_conf/patch/__init__.py
+++ b/dongtai_conf/patch/__init__.py
@@ -20,7 +20,7 @@ class PatchConfig:
 
 
 is_init_patch = False
-PATCH_HANDLER: dict[CodeType, tuple[Callable, PatchConfig]] = {}
+PATCH_HANDLER: dict[CodeType, dict[int, tuple[Callable, PatchConfig]]] = {}
 
 
 def init_patch() -> None:
@@ -39,16 +39,16 @@ def init_patch() -> None:
 
 
 @overload
-def patch_point(*args: Unpack[tuple[T]]) -> T:
+def patch_point(*args: Unpack[tuple[T]], patch_id: int = 0) -> T:
     ...
 
 
 @overload
-def patch_point(*args: Unpack[Ts]) -> tuple[Unpack[Ts]]:
+def patch_point(*args: Unpack[Ts], patch_id: int = 0) -> tuple[Unpack[Ts]]:
     ...
 
 
-def patch_point(*args: Any) -> Any:
+def patch_point(*args: Any, patch_id: int = 0) -> Any:
     init_patch()
     current_frame = inspect.currentframe()
     if current_frame is None:
@@ -60,7 +60,7 @@ def patch_point(*args: Any) -> Any:
         return _return_args(*args)
     caller_code = caller_frame.f_code
     if caller_code in PATCH_HANDLER:
-        func, patch_config = PATCH_HANDLER[caller_code]
+        func, patch_config = PATCH_HANDLER[caller_code][patch_id]
         func_args, _, _, _, kwonlyargs, _, annotations = inspect.getfullargspec(func)
         func_args += kwonlyargs
 
@@ -107,9 +107,12 @@ def _return_args(*args: Unpack[Ts]) -> tuple[Unpack[Ts]] | Any:
     return args
 
 
-def patch(patch_func: Callable, type_check: bool = False):
+def patch(patch_func: Callable, type_check: bool = False, patch_id: int = 0):
     def wrapper(func: Callable):
-        PATCH_HANDLER[patch_func.__code__] = (func, PatchConfig(type_check=type_check))
+        PATCH_HANDLER[patch_func.__code__][patch_id] = (
+            func,
+            PatchConfig(type_check=type_check),
+        )
         return func
 
     return wrapper

From 377efa8e85617bf323d6fe2913b54c1e5d663c8d Mon Sep 17 00:00:00 2001
From: st1020 
Date: Tue, 11 Jul 2023 11:06:22 +0800
Subject: [PATCH 097/161] feat: update patch

---
 dongtai_conf/patch/__init__.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/dongtai_conf/patch/__init__.py b/dongtai_conf/patch/__init__.py
index 9f833c234..7b6fa5d40 100644
--- a/dongtai_conf/patch/__init__.py
+++ b/dongtai_conf/patch/__init__.py
@@ -2,6 +2,7 @@
 import inspect
 import logging
 import pkgutil
+from collections import defaultdict
 from dataclasses import dataclass
 from pathlib import Path
 from types import CodeType
@@ -20,7 +21,9 @@ class PatchConfig:
 
 
 is_init_patch = False
-PATCH_HANDLER: dict[CodeType, dict[int, tuple[Callable, PatchConfig]]] = {}
+PATCH_HANDLER: dict[CodeType, dict[int, tuple[Callable, PatchConfig]]] = defaultdict(
+    dict
+)
 
 
 def init_patch() -> None:

From b8b8ae9246463f3ef9a06e17da9f102f2cfcc7e8 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Tue, 11 Jul 2023 11:29:01 +0800
Subject: [PATCH 098/161] feat: cut out vul list

---
 dongtai_web/aggr_vul/app_vul_list.py    | 34 +++-------
 dongtai_web/aggr_vul/app_vul_summary.py | 90 ++-----------------------
 2 files changed, 15 insertions(+), 109 deletions(-)

diff --git a/dongtai_web/aggr_vul/app_vul_list.py b/dongtai_web/aggr_vul/app_vul_list.py
index d042d5005..b05f5d44c 100644
--- a/dongtai_web/aggr_vul/app_vul_list.py
+++ b/dongtai_web/aggr_vul/app_vul_list.py
@@ -3,34 +3,29 @@
 from dongtai_common.endpoint import R
 from dongtai_common.endpoint import UserEndPoint
 from dongtai_common.models.vulnerablity import IastVulnerabilityModel
-from dongtai_web.aggregation.aggregation_common import turnIntListOfStr, auth_user_list_str
+from dongtai_web.aggregation.aggregation_common import turnIntListOfStr
 from dongtai_web.serializers.vul import VulSerializer
 from django.utils.translation import gettext_lazy as _
-from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
+from dongtai_web.utils import extend_schema_with_envcheck
 from dongtai_common.models.vulnerablity import IastVulnerabilityStatus
 import pymysql
 from dongtai_web.serializers.aggregation import AggregationArgsSerializer
-from dongtai_common.models import AGGREGATION_ORDER, LANGUAGE_ID_DICT, APP_LEVEL_RISK, APP_VUL_ORDER
+from dongtai_common.models import APP_LEVEL_RISK, APP_VUL_ORDER
 from django.db.models import F
 from dongtai_common.utils.db import SearchLanguageMode
 from dongtai_common.models.vulnerablity import IastVulnerabilityDocument
-from elasticsearch_dsl import Q, Search
+from elasticsearch_dsl import Q
 from elasticsearch import Elasticsearch
-from elasticsearch_dsl import A
-from dongtai_common.models.strategy import IastStrategyModel
-from dongtai_common.models.vulnerablity import IastVulnerabilityStatus
-from dongtai_common.models.program_language import IastProgramLanguage
-from dongtai_common.models.project import IastProject
-from dongtai_common.models.vul_level import IastVulLevel
 from django.core.cache import cache
 from dongtai_conf import settings
 from dongtai_common.common.utils import make_hash
 from dongtai_conf.settings import ELASTICSEARCH_STATE
 from dongtai_engine.elatic_search.data_correction import data_correction_interpetor
 from dongtai_common.models.dast_integration import IastDastIntegrationRelation
-from django.db.models import (Count, Sum)
+from django.db.models import (Count)
 from itertools import groupby
 from collections import defaultdict
+from dongtai_conf.patch import patch_point
 
 INT_LIMIT: int = 2**64 - 1
 
@@ -85,12 +80,7 @@ def post(self, request):
                             "project_version_id"))
                     es_query['project_version_id'] = ser.validated_data.get(
                         "project_version_id")
-                # 按项目筛选
-                if ser.validated_data.get("project_id_str", ""):
-                    project_id_list = turnIntListOfStr(
-                        ser.validated_data.get("project_id_str", ""))
-                    queryset = queryset.filter(project_id__in=project_id_list)
-                    es_query['project_ids'] = project_id_list
+                ser, queryset, es_query = patch_point(ser, queryset, es_query)
                 if ser.validated_data.get("uri", ""):
                     queryset = queryset.filter(
                         uri=ser.validated_data.get("uri", ""))
@@ -112,15 +102,7 @@ def post(self, request):
                         ser.validated_data.get("status_id_str", ""))
                     queryset = queryset.filter(status_id__in=status_id_list)
                     es_query['status_ids'] = status_id_list
-                # 按语言筛选
-                if ser.validated_data.get("language_str", ""):
-                    language_id_list = turnIntListOfStr(
-                        ser.validated_data.get("language_str", ""))
-                    language_arr = []
-                    for lang in language_id_list:
-                        language_arr.append(LANGUAGE_ID_DICT.get(str(lang)))
-                    queryset = queryset.filter(language__in=language_arr)
-                    es_query['language_ids'] = language_arr
+
                 order_list = []
                 fields = [
                     "id", "uri", "http_method", "top_stack", "bottom_stack",
diff --git a/dongtai_web/aggr_vul/app_vul_summary.py b/dongtai_web/aggr_vul/app_vul_summary.py
index 1f92fceb2..684e7796c 100644
--- a/dongtai_web/aggr_vul/app_vul_summary.py
+++ b/dongtai_web/aggr_vul/app_vul_summary.py
@@ -1,8 +1,7 @@
 from dongtai_common.models.department import Department
 from dongtai_web.utils import dict_transfrom
-from dongtai_engine.elatic_search.data_correction import data_correction_interpetor
 from dongtai_conf.settings import ELASTICSEARCH_STATE
-import copy
+
 from rest_framework.serializers import ValidationError
 from dongtai_common.endpoint import R
 from dongtai_common.endpoint import UserEndPoint
@@ -10,12 +9,10 @@
 from django.utils.translation import gettext_lazy as _
 from dongtai_web.utils import extend_schema_with_envcheck
 from dongtai_web.serializers.aggregation import AggregationArgsSerializer
-from dongtai_common.models import LANGUAGE_DICT
-from dongtai_web.aggregation.aggregation_common import auth_user_list_str
 from django.db.models import Count
-from dongtai_common.common.utils import cached_decorator
 from django.db.models import Q
 import logging
+from dongtai_conf.patch import patch_point
 logger = logging.getLogger('dongtai-webapi')
 
 
@@ -49,12 +46,6 @@ def get_annotate_data(
     # 项目版本号
     if project_version_id:
         cache_q = cache_q & Q(project_version_id=project_version_id)
-    # 项目统计
-    pro_info = _annotate_by_query(
-        cache_q,
-        ("project_id", "project__name"),
-        "project_id",
-    )
 
     result_summary = {
         "level": [],
@@ -64,14 +55,8 @@ def get_annotate_data(
         "project": [],
     }
 
-    for item in pro_info:
-        result_summary["project"].append(
-            {
-                "name": item["project__name"],
-                "num": item["count"],
-                "id": item["project_id"],
-            }
-        )
+    cache_q, result_summary = patch_point(cache_q, result_summary)
+
     # 漏洞类型统计
     strategy_info = _annotate_by_query(
         cache_q, ("strategy_id", "strategy__vul_name"), "strategy_id"
@@ -111,30 +96,6 @@ def get_annotate_data(
             }
         )
 
-    # # 按语言筛选
-    language_info = _annotate_by_query(
-        cache_q, ("language",), "language")
-    lang_arr = copy.copy(LANGUAGE_DICT)
-    lang_key = lang_arr.keys()
-    for item in language_info:
-        result_summary["language"].append(
-            {
-                "name": item["language"],
-                "num": item["count"],
-                "id": lang_arr.get(item["language"]),
-            }
-        )
-        if item["language"] in lang_key:
-            del lang_arr[item["language"]]
-    if lang_arr:
-        for item in lang_arr.keys():
-            result_summary["language"].append(
-                {
-                    "name": item,
-                    "num": 0,
-                    "id": LANGUAGE_DICT.get(item),
-                }
-            )
     return result_summary
 
 
@@ -190,13 +151,10 @@ def post(self, request):
 
 def get_annotate_data_es(department: Department, bind_project_id, project_version_id):
     from dongtai_common.models.vulnerablity import IastVulnerabilityDocument
-    from elasticsearch_dsl import Q, Search
     from elasticsearch import Elasticsearch
-    from elasticsearch_dsl import A
+    from elasticsearch_dsl import A, Q
     from dongtai_common.models.strategy import IastStrategyModel
     from dongtai_common.models.vulnerablity import IastVulnerabilityStatus
-    from dongtai_common.models.program_language import IastProgramLanguage
-    from dongtai_common.models.project import IastProject
     from dongtai_common.models.vul_level import IastVulLevel
 
     # user_id_list = [user_id]
@@ -221,11 +179,10 @@ def get_annotate_data_es(department: Department, bind_project_id, project_versio
         Q('bool', must=must_query))[:0]
     buckets = {
         'level': A('terms', field='level_id', size=2147483647),
-        'project': A('terms', field='bind_project_id', size=2147483647),
         "strategy": A('terms', field='strategy_id', size=2147483647),
         'status': A('terms', field='status_id', size=2147483647),
-        "language": A('terms', field='language.keyword', size=2147483647)
     }
+    buckets = patch_point(buckets, patch_id=0)
     for k, v in buckets.items():
         search.aggs.bucket(k, v)
     from dongtai_conf import settings
@@ -254,40 +211,6 @@ def get_annotate_data_es(department: Department, bind_project_id, project_versio
             status_dic = dict_transfrom(status, 'id')
             for i in origin_buckets:
                 i['name'] = status_dic[i['id']]['name']
-        if key == 'language':
-            for i in origin_buckets:
-                i['name'] = i['id']
-                del i['id']
-            language_names = [i['name'] for i in origin_buckets]
-            for i in origin_buckets:
-                i['id'] = LANGUAGE_DICT.get(i['name'])
-            for language_key in LANGUAGE_DICT.keys():
-                if language_key not in language_names:
-                    origin_buckets.append({
-                        'id': LANGUAGE_DICT[language_key],
-                        'name': language_key,
-                        'num': 0
-                    })
-        if key == 'project':
-            project_ids = [i['id'] for i in origin_buckets]
-            project = IastProject.objects.filter(pk__in=project_ids).values(
-                'id', 'name').all()
-            project_dic = dict_transfrom(project, 'id')
-            missing_ids = []
-            for i in origin_buckets:
-                if i['id'] not in project_dic:
-                    logger.info('found data consistency incorrect start ')
-                    data_correction_interpetor.delay("project_missing")
-                    missing_ids.append(i['id'])
-                    continue
-                else:
-                    i['name'] = project_dic[i['id']]['name']
-            origin_buckets = filter(lambda x: x['id'] not in missing_ids,
-                                    origin_buckets)
-            if missing_ids:
-                logger.info('found data consistency incorrect ')
-                data_correction_interpetor.delay("project_missing")
-
         if key == 'level':
             level_ids = [i['id'] for i in origin_buckets]
             level = IastVulLevel.objects.filter(pk__in=level_ids).values(
@@ -296,5 +219,6 @@ def get_annotate_data_es(department: Department, bind_project_id, project_versio
             for i in origin_buckets:
                 i['name'] = level_dic[i['id']]['name_value']
             origin_buckets = sorted(origin_buckets, key=lambda x: x['id'])
+        key, origin_buckets = patch_point(key, origin_buckets, patch_id=1)
         dic[key] = list(origin_buckets)
     return dict(dic)

From 6780331753b64f4b13d57693c0a828f3677e8b2f Mon Sep 17 00:00:00 2001
From: tscuite 
Date: Tue, 11 Jul 2023 12:04:29 +0800
Subject: [PATCH 099/161] feat: update Pipfile

---
 Pipfile      |  1 +
 Pipfile.lock | 51 ++++++++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 47 insertions(+), 5 deletions(-)

diff --git a/Pipfile b/Pipfile
index ee23e0b10..f2f9be895 100644
--- a/Pipfile
+++ b/Pipfile
@@ -86,6 +86,7 @@ django-silk = "*"
 types-python-dateutil = "*"
 shortuuid = "*"
 networkit = "*"
+flower = ">=2.0.0"
 
 [dev-packages]
 
diff --git a/Pipfile.lock b/Pipfile.lock
index 034b72ad9..45fc084c4 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "4fe4b731e86a03b2e7707d5996956c116300d8fe2186d79397b1ae205e46fcd3"
+            "sha256": "05195aa03741a7ea08e6ea1eea47b1ea2ecd38ddd44a846a2a9008fcf9e5acaa"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -421,11 +421,11 @@
         },
         "django-cors-headers": {
             "hashes": [
-                "sha256:36a8d7a6dee6a85f872fe5916cc878a36d0812043866355438dfeda0b20b6b78",
-                "sha256:88a4bfae24b6404dd0e0640203cb27704a2a57fd546a429e5d821dfa53dd1acf"
+                "sha256:9ada212b0e2efd4a5e339360ffc869cb21ac5605e810afe69f7308e577ea5bde",
+                "sha256:f9749c6410fe738278bc2b6ef17f05195bc7b251693c035752d8257026af024f"
             ],
             "index": "pypi",
-            "version": "==4.1.0"
+            "version": "==4.2.0"
         },
         "django-cprofile-middleware": {
             "hashes": [
@@ -643,6 +643,14 @@
             "markers": "python_version >= '3.6'",
             "version": "==1.1.0"
         },
+        "flower": {
+            "hashes": [
+                "sha256:5657785d728a54914256c34fd0551fe2d7152aab08062ebc645bf86b97b8aec5",
+                "sha256:571f9ed1c57a622e862de35eceb8a4244f023fbcfb7175f53e45ebe679f46d90"
+            ],
+            "index": "pypi",
+            "version": "==2.0.0"
+        },
         "gevent": {
             "hashes": [
                 "sha256:018f93de7d5318d2fb440f846839a4464738468c3476d5c9cf7da45bb71c18bd",
@@ -783,6 +791,14 @@
             "index": "pypi",
             "version": "==20.1.0"
         },
+        "humanize": {
+            "hashes": [
+                "sha256:7ca0e43e870981fa684acb5b062deb307218193bca1a01f2b2676479df849b3a",
+                "sha256:df7c429c2d27372b249d3f26eb53b07b166b661326e0325793e0a988082e3889"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==4.7.0"
+        },
         "id-validator": {
             "hashes": [
                 "sha256:90b73b89f807f8b387beaa7f1f86baa4a64fa5674df5db9ab1a7362433f8e3b8"
@@ -1142,7 +1158,7 @@
                 "sha256:70e991dd503e3f1956632c621946bb8e71012172db740bab75cd9d2a447283a3",
                 "sha256:bfa5d118f91f08ef4694d4aaab168c98efc91dab086aee587e71d66a7001701a"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_version >= '3'",
             "version": "==1.12.0"
         },
         "mypy": {
@@ -1356,6 +1372,14 @@
             "index": "pypi",
             "version": "==9.3.0"
         },
+        "prometheus-client": {
+            "hashes": [
+                "sha256:21e674f39831ae3f8acde238afd9a27a37d0d2fb5a28ea094f0ce25d2cbf2091",
+                "sha256:e537f37160f6807b8202a6fc4764cdd19bac5480ddd3e0d463c3002b34462101"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==0.17.1"
+        },
         "prompt-toolkit": {
             "hashes": [
                 "sha256:04505ade687dc26dc4284b1ad19a83be2f2afe83e7a828ace0c72f3a1df72aac",
@@ -1839,6 +1863,23 @@
             "markers": "python_version < '3.11'",
             "version": "==2.0.1"
         },
+        "tornado": {
+            "hashes": [
+                "sha256:05615096845cf50a895026f749195bf0b10b8909f9be672f50b0fe69cba368e4",
+                "sha256:0c325e66c8123c606eea33084976c832aa4e766b7dff8aedd7587ea44a604cdf",
+                "sha256:29e71c847a35f6e10ca3b5c2990a52ce38b233019d8e858b755ea6ce4dcdd19d",
+                "sha256:4b927c4f19b71e627b13f3db2324e4ae660527143f9e1f2e2fb404f3a187e2ba",
+                "sha256:5b17b1cf5f8354efa3d37c6e28fdfd9c1c1e5122f2cb56dac121ac61baa47cbe",
+                "sha256:6a0848f1aea0d196a7c4f6772197cbe2abc4266f836b0aac76947872cd29b411",
+                "sha256:7efcbcc30b7c654eb6a8c9c9da787a851c18f8ccd4a5a3a95b05c7accfa068d2",
+                "sha256:834ae7540ad3a83199a8da8f9f2d383e3c3d5130a328889e4cc991acc81e87a0",
+                "sha256:b46a6ab20f5c7c1cb949c72c1994a4585d2eaa0be4853f50a03b5031e964fc7c",
+                "sha256:c2de14066c4a38b4ecbbcd55c5cc4b5340eb04f1c5e81da7451ef555859c833f",
+                "sha256:c367ab6c0393d71171123ca5515c61ff62fe09024fa6bf299cd1339dc9456829"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==6.3.2"
+        },
         "types-awscrt": {
             "hashes": [
                 "sha256:5a846b77a6d35b63ccf9d33abf386c4dfc6275f9c062bc0e234c14ea8c6013a9",

From 91b38f9785260df9f1cf0c8315eaa651b4eac96e Mon Sep 17 00:00:00 2001
From: tscuite 
Date: Tue, 11 Jul 2023 12:07:52 +0800
Subject: [PATCH 100/161] feat: update ci

---
 .github/deploy/deploy-dongtai-server-test.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/deploy/deploy-dongtai-server-test.yml b/.github/deploy/deploy-dongtai-server-test.yml
index 75325fb12..e2fa268da 100644
--- a/.github/deploy/deploy-dongtai-server-test.yml
+++ b/.github/deploy/deploy-dongtai-server-test.yml
@@ -15,19 +15,19 @@ data:
     echo $1
     
     if [ "$1" = "worker" ]; then
-    	celery -A dongtai_conf worker -l info $DONGTAI_CONCURRENCY -E --pidfile= 
+    	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py worker -l info $DONGTAI_CONCURRENCY -E --pidfile= 
     elif [ "$1" = "worker-beat" ]; then
-    	celery -A dongtai_conf worker -l info -Q dongtai-periodic-task $DONGTAI_CONCURRENCY -E --pidfile=
+    	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py worker -l info -Q dongtai-periodic-task $DONGTAI_CONCURRENCY -E --pidfile=
     elif [ "$1" = "worker-high-freq" ]; then
-    	celery -A dongtai_conf worker -l info -Q dongtai-method-pool-scan,dongtai-replay-vul-scan $DONGTAI_CONCURRENCY -E --pidfile=
+    	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py worker -l info -Q dongtai-method-pool-scan,dongtai-replay-vul-scan $DONGTAI_CONCURRENCY -E --pidfile=
     elif [ "$1" = "worker-es" ]; then
-    	celery -A dongtai_conf worker -l info -Q dongtai-es-save-task $DONGTAI_CONCURRENCY -E --pidfile=  
+    	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py worker -l info -Q dongtai-es-save-task $DONGTAI_CONCURRENCY -E --pidfile=  
     elif [ "$1" = "worker-sca" ]; then
-    	celery -A dongtai_conf worker -l info -Q dongtai-sca-task $DONGTAI_CONCURRENCY -E --pidfile=  
+    	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py worker -l info -Q dongtai-sca-task $DONGTAI_CONCURRENCY -E --pidfile=  
     elif [ "$1" = "worker-other" ]; then
-    	celery -A dongtai_conf worker -l info -X dongtai-periodic-task,dongtai-method-pool-scan,dongtai-replay-vul-scan,dongtai-sca-task $DONGTAI_CONCURRENCY -E --pidfile= 
+    	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py worker -l info -X dongtai-periodic-task,dongtai-method-pool-scan,dongtai-replay-vul-scan,dongtai-sca-task $DONGTAI_CONCURRENCY -E --pidfile= 
     elif [ "$1" = "beat" ]; then
-    	celery -A dongtai_conf beat -l info $DONGTAI_CONCURRENCY  --pidfile= --scheduler django_celery_beat.schedulers:DatabaseScheduler
+    	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py beat -l info $DONGTAI_CONCURRENCY  --pidfile= --scheduler django_celery_beat.schedulers:DatabaseScheduler
     else
     	# echo "Get the latest vulnerability rules." && python manage.py load_hook_strategy 2> /dev/null
     	# if [ $? -ne 0 ]; then echo "ERROR: Lost connection to MySQL server !!!" && exit 1 ; else echo "succeed" ;fi

From 9f4ea2755dcc39fd3f4f6c2706e1f196c58552e4 Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Tue, 11 Jul 2023 12:19:55 +0800
Subject: [PATCH 101/161] feat: project version timestamp update.

---
 dongtai_engine/plugins/project_time_update.py        | 4 ----
 dongtai_engine/signals/handlers/vul_handler.py       | 4 ++++
 dongtai_engine/tasks.py                              | 7 ++++++-
 dongtai_protocol/report/handler/api_route_handler.py | 7 ++++++-
 4 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/dongtai_engine/plugins/project_time_update.py b/dongtai_engine/plugins/project_time_update.py
index 286bfbfa6..e1b7083c4 100644
--- a/dongtai_engine/plugins/project_time_update.py
+++ b/dongtai_engine/plugins/project_time_update.py
@@ -24,7 +24,3 @@ def project_version_time_stamp_update(project_version_id):
     timestamp = int(time())
     IastProjectVersion.objects.filter(pk=project_version_id).update(
         update_time=timestamp)
-from dongtai_engine.plugins.project_time_update import (
-    project_time_stamp_update,
-    project_version_time_stamp_update,
-)
diff --git a/dongtai_engine/signals/handlers/vul_handler.py b/dongtai_engine/signals/handlers/vul_handler.py
index eeebb2430..d921f1998 100644
--- a/dongtai_engine/signals/handlers/vul_handler.py
+++ b/dongtai_engine/signals/handlers/vul_handler.py
@@ -291,6 +291,8 @@ def save_vul(vul_meta, vul_level, strategy_id, vul_stack, top_stack,
     ).order_by('-latest_time').first()
     project_time_stamp_update.apply_async(
         (vul_meta.agent.bind_project_id, ), countdown=5)
+    project_version_time_stamp_update.apply_async(
+        (vul_meta.agent.project_version_id, ), countdown=5)
     if vul:
         vul.url = vul_meta.url
         vul.uri = vul_meta.uri
@@ -444,6 +446,8 @@ def handler_replay_vul(vul_meta, vul_level, strategy_id, vul_stack, top_stack,
             update_fields=['status_id', 'latest_time', 'latest_time_desc'])
         project_time_stamp_update.apply_async(
             (vul_meta.agent.bind_project_id, ), countdown=5)
+        project_version_time_stamp_update.apply_async(
+            (vul_meta.agent.project_version_id, ), countdown=5)
 
         IastReplayQueue.objects.filter(id=kwargs['replay_id']).update(
             state=const.SOLVED,
diff --git a/dongtai_engine/tasks.py b/dongtai_engine/tasks.py
index 3f1069fe9..4fe569c81 100644
--- a/dongtai_engine/tasks.py
+++ b/dongtai_engine/tasks.py
@@ -38,7 +38,10 @@
 import requests
 from dongtai_engine.task_base import replay_payload_data
 from dongtai_engine.common.queryset import get_scan_id, load_sink_strategy, get_agent
-from dongtai_engine.plugins.project_time_update import project_time_stamp_update
+from dongtai_engine.plugins.project_time_update import (
+    project_time_stamp_update,
+    project_version_time_stamp_update,
+)
 from dongtai_web.vul_log.vul_log import log_recheck_vul
 
 RETRY_INTERVALS = [10, 30, 90]
@@ -161,6 +164,8 @@ def search_and_save_vul(engine: Optional[VulEngine],
                 update_time=timestamp)
             project_time_stamp_update.apply_async(
                 (method_pool_model.agent.bind_project_id, ), countdown=5)
+            project_version_time_stamp_update.apply_async(
+                (method_pool_model.agent.project_version_id, ), countdown=5)
         except Exception as e:
             logger.info(f'漏洞数据处理出错，原因：{e}')
 
diff --git a/dongtai_protocol/report/handler/api_route_handler.py b/dongtai_protocol/report/handler/api_route_handler.py
index 20859ccab..8b8a9c0ff 100644
--- a/dongtai_protocol/report/handler/api_route_handler.py
+++ b/dongtai_protocol/report/handler/api_route_handler.py
@@ -17,7 +17,10 @@
 from django.utils.translation import gettext_lazy as _
 from django.db import transaction
 from dongtai_common.models.project import IastProject
-from dongtai_engine.plugins.project_time_update import project_time_stamp_update
+from dongtai_engine.plugins.project_time_update import (
+    project_time_stamp_update,
+    project_version_time_stamp_update,
+)
 from celery import shared_task
 
 logger = logging.getLogger('dongtai.openapi')
@@ -81,6 +84,8 @@ def api_route_gather(agent_id, api_routes):
             logger.info(_('API navigation log record successfully'))
         project_time_stamp_update.apply_async((agent.bind_project_id, ),
                                               countdown=5)
+        project_version_time_stamp_update.apply_async(
+            (agent.project_version_id, ), countdown=5)
     except Exception as e:
         logger.info(_('API navigation log failed, why: {}').format(e),
                     exc_info=e)

From 0705afe07b5bdf108031d4941854d814cde96982 Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Tue, 11 Jul 2023 14:41:41 +0800
Subject: [PATCH 102/161] feat: project version timestamp update.

---
 dongtai_web/views/project_summary.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/dongtai_web/views/project_summary.py b/dongtai_web/views/project_summary.py
index 974a9643e..713e8340d 100644
--- a/dongtai_web/views/project_summary.py
+++ b/dongtai_web/views/project_summary.py
@@ -8,6 +8,7 @@
 from dongtai_common.endpoint import UserEndPoint
 from dongtai_common.models.agent import IastAgent
 from dongtai_common.models.project import IastProject
+from dongtai_common.models.project_version import IastProjectVersion
 from dongtai_common.models.vul_level import IastVulLevel
 from dongtai_common.models.vulnerablity import IastVulnerabilityModel
 from dongtai_web.base.project_version import get_project_version, get_project_version_by_id, ProjectsVersionDataSerializer
@@ -119,8 +120,7 @@ def get(self, request, id):
         data['level_count'] = []
 
         if not version_id:
-            current_project_version = get_project_version(
-                project.id)
+            current_project_version = get_project_version(project.id)
         else:
             current_project_version = get_project_version_by_id(version_id)
         data['versionData'] = current_project_version
@@ -134,4 +134,9 @@ def get(self, request, id):
             project).data['agent_language']
         data['agent_alive'] = IastAgent.objects.filter(
             bind_project_id=project.id, online=const.RUNNING).count()
+        project_version = IastProjectVersion.objects.filter(
+            pk=current_project_version.get("version_id", 0)).first()
+        data[
+            'project_version_latest_time'] = project_version.update_time if project_version else project.latest_time
+        data['type_summary'] = []
         return R.success(data=data)

From 407ed1379beab7922804034b209e261107ae4fde Mon Sep 17 00:00:00 2001
From: tscuite 
Date: Tue, 11 Jul 2023 16:02:34 +0800
Subject: [PATCH 103/161] feat: add metrics

---
 Pipfile                     |  4 +++-
 Pipfile.lock                | 24 ++++++++++++++++++++----
 deploy/docker/entrypoint.sh | 14 +++++++-------
 dongtai_conf/settings.py    |  9 +++++++++
 dongtai_conf/urls.py        |  4 ++++
 requirements.txt            | 12 +++++++++---
 6 files changed, 52 insertions(+), 15 deletions(-)

diff --git a/Pipfile b/Pipfile
index f2f9be895..55486ba7b 100644
--- a/Pipfile
+++ b/Pipfile
@@ -86,7 +86,9 @@ django-silk = "*"
 types-python-dateutil = "*"
 shortuuid = "*"
 networkit = "*"
-flower = ">=2.0.0"
+flower = "~=2.0.0"
+django-health-check = "==3.17.0"
+django-prometheus = "==2.3.1"
 
 [dev-packages]
 
diff --git a/Pipfile.lock b/Pipfile.lock
index 45fc084c4..ee1466023 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "05195aa03741a7ea08e6ea1eea47b1ea2ecd38ddd44a846a2a9008fcf9e5acaa"
+            "sha256": "bd16237109411b522dc3f42ae2bcf1c44ddb2dec4f74aee4ef66f17e55794fa4"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -101,11 +101,11 @@
         },
         "boto3-stubs": {
             "hashes": [
-                "sha256:4595eebb4a92808a26b827db31e4ded3d1f9caed64c539d468c4ff2f327badc2",
-                "sha256:5fb848352946deee89f260f062f8a8238b282a39e5bf36edf8d79aa72af2a143"
+                "sha256:b140f56315cd99c659a2cbae32dc4ae1ee44073b4250e1ad391d03ecf4b5eb40",
+                "sha256:bcef1fcbd758de6078e75b036d3632dd95eaef00311e6688554b5b883a194563"
             ],
             "index": "pypi",
-            "version": "==1.28.1"
+            "version": "==1.28.2"
         },
         "botocore": {
             "hashes": [
@@ -450,6 +450,14 @@
             "index": "pypi",
             "version": "==23.2"
         },
+        "django-health-check": {
+            "hashes": [
+                "sha256:20dc5ccb516a4e7163593fd4026f0a7531e3027b47d23ebe3bd9dbc99ac4354c",
+                "sha256:d1b8671e79d1de6e3dd1a9c69566222b0bfcfacca8b90511a4407b2d0d3d2778"
+            ],
+            "index": "pypi",
+            "version": "==3.17.0"
+        },
         "django-import-export": {
             "hashes": [
                 "sha256:c39c003bfc803fb63ba7742562f1667603a4a8d7426261845d75ce8582d40f48",
@@ -472,6 +480,14 @@
             "index": "pypi",
             "version": "==0.17.7"
         },
+        "django-prometheus": {
+            "hashes": [
+                "sha256:cf9b26f7ba2e4568f08f8f91480a2882023f5908579681bcf06a4d2465f12168",
+                "sha256:f9c8b6c780c9419ea01043c63a437d79db2c33353451347894408184ad9c3e1e"
+            ],
+            "index": "pypi",
+            "version": "==2.3.1"
+        },
         "django-ranged-response": {
             "hashes": [
                 "sha256:f71fff352a37316b9bead717fc76e4ddd6c9b99c4680cdf4783b9755af1cf985"
diff --git a/deploy/docker/entrypoint.sh b/deploy/docker/entrypoint.sh
index b9835781d..f021ab892 100755
--- a/deploy/docker/entrypoint.sh
+++ b/deploy/docker/entrypoint.sh
@@ -7,19 +7,19 @@ python -c "import deploy.docker.version_update" || true
 echo $1
 
 if [ "$1" = "worker" ]; then
-	celery -A dongtai_conf worker -l info $DONGTAI_CONCURRENCY -E --pidfile= 
+	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py worker -l info $DONGTAI_CONCURRENCY -E --pidfile=
 elif [ "$1" = "worker-beat" ]; then
-	celery -A dongtai_conf worker -l info -Q dongtai-periodic-task $DONGTAI_CONCURRENCY -E --pidfile=
+	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py worker -l info -Q dongtai-periodic-task $DONGTAI_CONCURRENCY -E --pidfile=
 elif [ "$1" = "worker-high-freq" ]; then
-	celery -A dongtai_conf worker -l info -Q dongtai-method-pool-scan,dongtai-replay-vul-scan $DONGTAI_CONCURRENCY -E --pidfile=
+	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py worker -l info -Q dongtai-method-pool-scan,dongtai-replay-vul-scan $DONGTAI_CONCURRENCY -E --pidfile=
 elif [ "$1" = "worker-es" ]; then
-	celery -A dongtai_conf worker -l info -Q dongtai-es-save-task $DONGTAI_CONCURRENCY -E --pidfile=  
+	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py worker -l info -Q dongtai-es-save-task $DONGTAI_CONCURRENCY -E --pidfile=
 elif [ "$1" = "worker-sca" ]; then
-	celery -A dongtai_conf worker -l info -Q dongtai-sca-task,dongtai-api-route-handler $DONGTAI_CONCURRENCY -E --pidfile=  
+	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py worker -l info -Q dongtai-sca-task,dongtai-api-route-handler $DONGTAI_CONCURRENCY -E --pidfile=
 elif [ "$1" = "worker-other" ]; then
-	celery -A dongtai_conf worker -l info -X dongtai-periodic-task,dongtai-method-pool-scan,dongtai-replay-vul-scan,dongtai-sca-task $DONGTAI_CONCURRENCY -E --pidfile= 
+	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py worker -l info -X dongtai-periodic-task,dongtai-method-pool-scan,dongtai-replay-vul-scan,dongtai-sca-task $DONGTAI_CONCURRENCY -E --pidfile=
 elif [ "$1" = "beat" ]; then
-	celery -A dongtai_conf beat -l info $DONGTAI_CONCURRENCY  --pidfile= --scheduler django_celery_beat.schedulers:DatabaseScheduler
+	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py beat -l info $DONGTAI_CONCURRENCY  --pidfile= --scheduler django_celery_beat.schedulers:DatabaseScheduler
 else
 	echo "Get the latest vulnerability rules." && python manage.py load_hook_strategy
 	if [ $? -ne 0 ]; then echo "ERROR: Lost connection to MySQL server !!!" && exit 1 ; else echo "succeed" ;fi
diff --git a/dongtai_conf/settings.py b/dongtai_conf/settings.py
index d4dbb66a3..eed0777ea 100644
--- a/dongtai_conf/settings.py
+++ b/dongtai_conf/settings.py
@@ -82,6 +82,10 @@ def ranstr(num):
     'django_celery_beat',
     'deploy.commands',
     'test.debug',
+    'health_check',                             # required
+    'health_check.db',                          # stock Django health checkers
+    'health_check.contrib.redis',
+    'django_prometheus',
 ]
 DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
 
@@ -156,6 +160,11 @@ def get_installed_apps():
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
 ]
 
+PROMETHEUS_LATENCY_BUCKETS = (0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1.0, 2.5, 5.0, 7.5, 10.0, 25.0, 50.0, 75.0, float("inf"),)
+if os.getenv('METRICS', None) == 'true':
+    MIDDLEWARE.extend(['django_prometheus.middleware.PrometheusBeforeMiddleware',
+                       'django_prometheus.middleware.PrometheusAfterMiddleware'
+                       ])
 try:
     from dongtai_conf.settings_extend import MIDDLEWARE as MIDDLEWARE_EXTEND
 
diff --git a/dongtai_conf/urls.py b/dongtai_conf/urls.py
index 65aa5a30f..2818bba34 100644
--- a/dongtai_conf/urls.py
+++ b/dongtai_conf/urls.py
@@ -24,6 +24,9 @@
     path('', include('{}.urls'.format(app))) for app in settings.CUSTOM_APPS
 ]
 urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
+urlpatterns.extend([path('healthcheck', include('health_check.urls'))])
+if os.getenv('METRICS', None) == 'true':
+    urlpatterns.extend([path('', include('django_prometheus.urls'))])
 
 if os.getenv('environment', 'PROD') in ('TEST', 'DOC') or os.getenv('DOC', None) == 'TRUE':
     from drf_spectacular.views import SpectacularJSONAPIView, SpectacularRedocView, SpectacularSwaggerView
@@ -39,6 +42,7 @@
              name='redoc'),
     ])
 
+
 if os.getenv('DJANGOSILK', None) == 'TRUE':
     silk_path = os.getenv(
         'DJANGOSILKPATH',
diff --git a/requirements.txt b/requirements.txt
index d03531626..ff4536c5c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,7 +10,7 @@ attrs==23.1.0 ; python_version >= '3.7'
 autopep8==2.0.2 ; python_version >= '3.6'
 billiard==4.1.0 ; python_version >= '3.7'
 boto3==1.24.59
-boto3-stubs==1.28.1
+boto3-stubs==1.28.2
 botocore==1.27.91
 botocore-stubs==1.29.165
 celery==5.3.0rc1
@@ -31,13 +31,15 @@ defusedxml==0.7.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1,
 diff-match-patch==20230430 ; python_version >= '3.7'
 django==3.2.20
 django-celery-beat==2.2.0
-django-cors-headers==4.1.0
+django-cors-headers==4.2.0
 django-cprofile-middleware==1.0.5
 django-elasticsearch-dsl==7.2.2
 django-filter==23.2
+django-health-check==3.17.0
 django-import-export==2.5.0
 django-mock-queries==v2.1.7
 django-modeltranslation==0.17.7
+django-prometheus==2.3.1
 django-ranged-response==0.2.0
 django-redis==5.2.0
 django-rest-framework-proxy==1.6.0
@@ -59,10 +61,12 @@ drf-spectacular==0.22.1
 elasticsearch==7.17.7
 elasticsearch-dsl==7.4.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 et-xmlfile==1.1.0 ; python_version >= '3.6'
+flower==2.0.0
 gevent==22.10.2
 gprof2dot==2022.7.29 ; python_version >= '2.7'
 greenlet==2.0.2 ; platform_python_implementation == 'CPython'
 gunicorn==20.1.0
+humanize==4.7.0 ; python_version >= '3.8'
 id-validator==1.0.20
 idna==2.10
 inflection==0.5.1 ; python_version >= '3.5'
@@ -79,7 +83,7 @@ markupsafe==2.1.3 ; python_version >= '3.7'
 marshmallow==3.19.0 ; python_version >= '3.7'
 marshmallow-enum==1.5.1
 mock==5.0.2 ; python_version >= '3.6'
-model-bakery==1.12.0 ; python_version >= '3.7'
+model-bakery==1.12.0 ; python_version >= '3'
 mypy==1.0.1
 mypy-extensions==1.0.0 ; python_version >= '3.5'
 mysqlclient==2.2.0
@@ -91,6 +95,7 @@ oss2==2.13.1
 packaging==21.3
 pep8==1.7.1
 pillow==9.3.0
+prometheus-client==0.17.1 ; python_version >= '3.6'
 prompt-toolkit==3.0.39 ; python_full_version >= '3.7.0'
 pycodestyle==2.10.0 ; python_version >= '3.6'
 pycparser==2.21
@@ -120,6 +125,7 @@ six==1.15.0
 sqlparse==0.4.4 ; python_version >= '3.5'
 tablib[html,ods,xls,xlsx,yaml]==3.5.0 ; python_version >= '3.8'
 tomli==2.0.1 ; python_version < '3.11'
+tornado==6.3.2 ; python_version >= '3.8'
 types-awscrt==0.16.23 ; python_version >= '3.7' and python_version < '4.0'
 types-pymysql==1.1.0.0
 types-pyopenssl==23.2.0.1

From b95d5a7b931b82e9df9fd355ca4a10a681fb4d87 Mon Sep 17 00:00:00 2001
From: tscuite 
Date: Tue, 11 Jul 2023 16:29:23 +0800
Subject: [PATCH 104/161] feat: add metrics

---
 .github/deploy/deploy-dongtai-server-test.yml | 212 ++++++++++++++++++
 1 file changed, 212 insertions(+)

diff --git a/.github/deploy/deploy-dongtai-server-test.yml b/.github/deploy/deploy-dongtai-server-test.yml
index e2fa268da..743ddde34 100644
--- a/.github/deploy/deploy-dongtai-server-test.yml
+++ b/.github/deploy/deploy-dongtai-server-test.yml
@@ -74,6 +74,38 @@ spec:
           env:
             - name: DONGTAI_CONCURRENCY
               value: --processes 2 --stats :3031 --stats-http
+            - name: METRICS
+              value: =true
+          livenessProbe:
+            httpGet:
+              port: 8000
+              scheme: HTTP
+              path: /healthcheck
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            successThreshold: 1
+            timeoutSeconds: 3
+          readinessProbe:
+            httpGet:
+              port: 8000
+              scheme: HTTP
+              path: /healthcheck
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            successThreshold: 1
+            timeoutSeconds: 3
+          startupProbe:
+            httpGet:
+              port: 8000
+              scheme: HTTP
+              path: /healthcheck
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            successThreshold: 1
+            timeoutSeconds: 3
           resources:
             limits:
               cpu: "500m"
@@ -142,6 +174,36 @@ spec:
             - name: configfile
               mountPath: /opt/dongtai/dongtai_conf/conf/config.ini
               subPath: config.ini
+          livenessProbe:
+            httpGet:
+              port: 5555
+              scheme: HTTP
+              path: /healthcheck
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            successThreshold: 1
+            timeoutSeconds: 3
+          readinessProbe:
+            httpGet:
+              port: 5555
+              scheme: HTTP
+              path: /healthcheck
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            successThreshold: 1
+            timeoutSeconds: 3
+          startupProbe:
+            httpGet:
+              port: 5555
+              scheme: HTTP
+              path: /healthcheck
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            successThreshold: 1
+            timeoutSeconds: 3
           resources:
             limits:
               cpu: "500m"
@@ -192,6 +254,36 @@ spec:
           env:
             - name: DONGTAI_CONCURRENCY
               value: -P gevent --concurrency=121
+          livenessProbe:
+            httpGet:
+              port: 5555
+              scheme: HTTP
+              path: /healthcheck
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            successThreshold: 1
+            timeoutSeconds: 3
+          readinessProbe:
+            httpGet:
+              port: 5555
+              scheme: HTTP
+              path: /healthcheck
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            successThreshold: 1
+            timeoutSeconds: 3
+          startupProbe:
+            httpGet:
+              port: 5555
+              scheme: HTTP
+              path: /healthcheck
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            successThreshold: 1
+            timeoutSeconds: 3
           resources:
             limits:
               cpu: "500m"
@@ -241,6 +333,36 @@ spec:
           env:
             - name: DONGTAI_CONCURRENCY
               value: --concurrency=2
+          livenessProbe:
+            httpGet:
+              port: 5555
+              scheme: HTTP
+              path: /healthcheck
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            successThreshold: 1
+            timeoutSeconds: 3
+          readinessProbe:
+            httpGet:
+              port: 5555
+              scheme: HTTP
+              path: /healthcheck
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            successThreshold: 1
+            timeoutSeconds: 3
+          startupProbe:
+            httpGet:
+              port: 5555
+              scheme: HTTP
+              path: /healthcheck
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            successThreshold: 1
+            timeoutSeconds: 3
           resources:
             limits:
               cpu: "500m"
@@ -290,6 +412,36 @@ spec:
           env:
             - name: DONGTAI_CONCURRENCY
               value: --concurrency=2
+          livenessProbe:
+            httpGet:
+              port: 5555
+              scheme: HTTP
+              path: /healthcheck
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            successThreshold: 1
+            timeoutSeconds: 3
+          readinessProbe:
+            httpGet:
+              port: 5555
+              scheme: HTTP
+              path: /healthcheck
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            successThreshold: 1
+            timeoutSeconds: 3
+          startupProbe:
+            httpGet:
+              port: 5555
+              scheme: HTTP
+              path: /healthcheck
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            successThreshold: 1
+            timeoutSeconds: 3
           resources:
             limits:
               cpu: "500m"
@@ -339,6 +491,36 @@ spec:
           env:
             - name: DONGTAI_CONCURRENCYW
               value: -P gevent --concurrency=10
+          livenessProbe:
+            httpGet:
+              port: 5555
+              scheme: HTTP
+              path: /healthcheck
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            successThreshold: 1
+            timeoutSeconds: 3
+          readinessProbe:
+            httpGet:
+              port: 5555
+              scheme: HTTP
+              path: /healthcheck
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            successThreshold: 1
+            timeoutSeconds: 3
+          startupProbe:
+            httpGet:
+              port: 5555
+              scheme: HTTP
+              path: /healthcheck
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            successThreshold: 1
+            timeoutSeconds: 3
           resources:
             limits:
               cpu: "500m"
@@ -388,6 +570,36 @@ spec:
           env:
             - name: DONGTAI_CONCURRENCY
               value: -P gevent --concurrency=128
+          livenessProbe:
+            httpGet:
+              port: 5555
+              scheme: HTTP
+              path: /healthcheck
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            successThreshold: 1
+            timeoutSeconds: 3
+          readinessProbe:
+            httpGet:
+              port: 5555
+              scheme: HTTP
+              path: /healthcheck
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            successThreshold: 1
+            timeoutSeconds: 3
+          startupProbe:
+            httpGet:
+              port: 5555
+              scheme: HTTP
+              path: /healthcheck
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            successThreshold: 1
+            timeoutSeconds: 3
           resources:
             limits:
               cpu: "500m"

From 164f0c2cdee41bf9c497538869a2e956dbe730c6 Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Tue, 11 Jul 2023 16:29:55 +0800
Subject: [PATCH 105/161] fix: disable branch search.

---
 dongtai_common/engine/vul_engine.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/dongtai_common/engine/vul_engine.py b/dongtai_common/engine/vul_engine.py
index 8e1d9cad7..85df6541b 100644
--- a/dongtai_common/engine/vul_engine.py
+++ b/dongtai_common/engine/vul_engine.py
@@ -178,10 +178,10 @@ def search(self, method_pool, vul_method_signature, vul_type=None):
                 for path_key in total_path:
                     sub_method = invokeid_dict[path_key]
                     if sub_method.get('source'):
+                        self.vul_source_signature = f"{sub_method.get('className')}.{sub_method.get('methodName')}"
                         final_stack.append(
                             self.copy_method(sub_method, source=True))
                     if sub_method['invokeId'] == t:
-                        self.vul_source_signature = f"{sub_method.get('className')}.{sub_method.get('methodName')}"
                         self.taint_value = sub_method['targetValues']
                         final_stack.append(
                             self.copy_method(sub_method, sink=True))
@@ -189,7 +189,6 @@ def search(self, method_pool, vul_method_signature, vul_type=None):
                         final_stack.append(
                             self.copy_method(sub_method, propagator=True))
                 self.vul_stack = [final_stack]
-        current_link = list()
         if self.vul_source_signature and 'sourceType' in self.vul_stack[-1][
                 -1].keys():
             final_stack = self.vul_stack[-1][-1]
@@ -266,10 +265,12 @@ def search(self, method_pool, vul_method_signature, vul_type=None):
                 self.vul_stack = []
                 self.pool_value = ""
                 return
-            current_link = current_link[0:1]
-            extract_stack = self.find_other_branch_v2(
-                index, size, current_link, set(final_stack.get('sourceHash')))
-            self.vul_stack[0] = extract_stack[::-1]
+            #Disable temporary , will refactor it in next version.
+            #
+            #current_link = current_link[0:1]
+            #extract_stack = self.find_other_branch_v2(
+            #    index, size, current_link, set(final_stack.get('sourceHash')))
+            #self.vul_stack[0] = extract_stack[::-1]
         else:
             pass
         self.vul_filter()

From 0ca14583231c38216abb583b9254e1b7483dc754 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Tue, 11 Jul 2023 16:42:10 +0800
Subject: [PATCH 106/161] feat: change default strategy template when add
 strategy

---
 dongtai_web/views/strategys.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/dongtai_web/views/strategys.py b/dongtai_web/views/strategys.py
index aff7a252a..f15813017 100644
--- a/dongtai_web/views/strategys.py
+++ b/dongtai_web/views/strategys.py
@@ -4,6 +4,7 @@
 
 # software: PyCharm
 # project: lingzhi-webapi
+from dongtai_common.models.strategy_user import IastStrategyUser
 from dongtai_common.utils import const
 from dongtai_common.models.hook_type import HookType
 from dongtai_common.models.strategy import IastStrategyModel
@@ -197,6 +198,10 @@ def post(self, request):
                                                     user=request.user,
                                                     dt=time.time())
         strategy.save()
+        content = IastStrategyUser.objects.get(pk=5).content.split(",")
+        if str(strategy.id) not in content:
+            content.append(str(strategy.id))
+        IastStrategyUser.objects.filter(pk=5).update(content=",".join(content))
         for language in IastProgramLanguage.objects.all():
             HookType.objects.create(type=3,
                                     name=ser.validated_data['vul_name'],

From d712a5da2425e1f39b656fd77bdb27f2d3232955 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Tue, 11 Jul 2023 17:15:47 +0800
Subject: [PATCH 107/161] fix: vul status error in heartbeat

---
 dongtai_protocol/report/handler/heartbeat_handler.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/dongtai_protocol/report/handler/heartbeat_handler.py b/dongtai_protocol/report/handler/heartbeat_handler.py
index a974da603..91ea209ae 100644
--- a/dongtai_protocol/report/handler/heartbeat_handler.py
+++ b/dongtai_protocol/report/handler/heartbeat_handler.py
@@ -19,6 +19,7 @@
 from dongtai_common.models.project import VulValidation
 from dongtai_common.utils.systemsettings import get_vul_validate
 from django.core.cache import cache
+from dongtai_web.vul_log.vul_log import log_recheck_vul
 
 logger = logging.getLogger('dongtai.openapi')
 
@@ -151,9 +152,12 @@ def get_result(self, msg=None):
                 IastReplayQueue.objects.filter(id__in=failure_ids).update(
                     update_time=timestamp, state=const.SOLVED)
 
-                IastVulnerabilityModel.objects.filter(
-                    id__in=success_vul_ids).update(latest_time=timestamp,
-                                                   status_id=2)
+                log_recheck_vul(
+                    self.agent.user.id,
+                    self.agent.user.username,
+                    success_vul_ids,
+                    "验证中",
+                )
                 IastVulnerabilityModel.objects.filter(
                     id__in=failure_vul_ids).update(latest_time=timestamp,
                                                    status_id=1)

From 7a099e82ad28da40da9b88ea6954f2057fe456ea Mon Sep 17 00:00:00 2001
From: tscuite 
Date: Tue, 11 Jul 2023 17:36:10 +0800
Subject: [PATCH 108/161] feat: del metrics

---
 .github/deploy/deploy-dongtai-server-test.yml | 200 ++----------------
 deploy/docker/entrypoint.sh                   |  14 +-
 2 files changed, 20 insertions(+), 194 deletions(-)

diff --git a/.github/deploy/deploy-dongtai-server-test.yml b/.github/deploy/deploy-dongtai-server-test.yml
index 743ddde34..8425ab68d 100644
--- a/.github/deploy/deploy-dongtai-server-test.yml
+++ b/.github/deploy/deploy-dongtai-server-test.yml
@@ -15,19 +15,19 @@ data:
     echo $1
     
     if [ "$1" = "worker" ]; then
-    	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py worker -l info $DONGTAI_CONCURRENCY -E --pidfile= 
+    	celery -A dongtai_conf worker -l info $DONGTAI_CONCURRENCY -E --pidfile= 
     elif [ "$1" = "worker-beat" ]; then
-    	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py worker -l info -Q dongtai-periodic-task $DONGTAI_CONCURRENCY -E --pidfile=
+    	celery -A dongtai_conf worker -l info -Q dongtai-periodic-task $DONGTAI_CONCURRENCY -E --pidfile=
     elif [ "$1" = "worker-high-freq" ]; then
-    	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py worker -l info -Q dongtai-method-pool-scan,dongtai-replay-vul-scan $DONGTAI_CONCURRENCY -E --pidfile=
+    	celery -A dongtai_conf worker -l info -Q dongtai-method-pool-scan,dongtai-replay-vul-scan $DONGTAI_CONCURRENCY -E --pidfile=
     elif [ "$1" = "worker-es" ]; then
-    	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py worker -l info -Q dongtai-es-save-task $DONGTAI_CONCURRENCY -E --pidfile=  
+    	celery -A dongtai_conf worker -l info -Q dongtai-es-save-task $DONGTAI_CONCURRENCY -E --pidfile=  
     elif [ "$1" = "worker-sca" ]; then
-    	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py worker -l info -Q dongtai-sca-task $DONGTAI_CONCURRENCY -E --pidfile=  
+    	celery -A dongtai_conf worker -l info -Q dongtai-sca-task $DONGTAI_CONCURRENCY -E --pidfile=  
     elif [ "$1" = "worker-other" ]; then
-    	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py worker -l info -X dongtai-periodic-task,dongtai-method-pool-scan,dongtai-replay-vul-scan,dongtai-sca-task $DONGTAI_CONCURRENCY -E --pidfile= 
+    	celery -A dongtai_conf worker -l info -X dongtai-periodic-task,dongtai-method-pool-scan,dongtai-replay-vul-scan,dongtai-sca-task $DONGTAI_CONCURRENCY -E --pidfile= 
     elif [ "$1" = "beat" ]; then
-    	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py beat -l info $DONGTAI_CONCURRENCY  --pidfile= --scheduler django_celery_beat.schedulers:DatabaseScheduler
+    	celery -A dongtai_conf beat -l info $DONGTAI_CONCURRENCY  --pidfile= --scheduler django_celery_beat.schedulers:DatabaseScheduler
     else
     	# echo "Get the latest vulnerability rules." && python manage.py load_hook_strategy 2> /dev/null
     	# if [ $? -ne 0 ]; then echo "ERROR: Lost connection to MySQL server !!!" && exit 1 ; else echo "succeed" ;fi
@@ -174,36 +174,7 @@ spec:
             - name: configfile
               mountPath: /opt/dongtai/dongtai_conf/conf/config.ini
               subPath: config.ini
-          livenessProbe:
-            httpGet:
-              port: 5555
-              scheme: HTTP
-              path: /healthcheck
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            successThreshold: 1
-            timeoutSeconds: 3
-          readinessProbe:
-            httpGet:
-              port: 5555
-              scheme: HTTP
-              path: /healthcheck
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            successThreshold: 1
-            timeoutSeconds: 3
-          startupProbe:
-            httpGet:
-              port: 5555
-              scheme: HTTP
-              path: /healthcheck
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            successThreshold: 1
-            timeoutSeconds: 3
+
           resources:
             limits:
               cpu: "500m"
@@ -254,36 +225,7 @@ spec:
           env:
             - name: DONGTAI_CONCURRENCY
               value: -P gevent --concurrency=121
-          livenessProbe:
-            httpGet:
-              port: 5555
-              scheme: HTTP
-              path: /healthcheck
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            successThreshold: 1
-            timeoutSeconds: 3
-          readinessProbe:
-            httpGet:
-              port: 5555
-              scheme: HTTP
-              path: /healthcheck
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            successThreshold: 1
-            timeoutSeconds: 3
-          startupProbe:
-            httpGet:
-              port: 5555
-              scheme: HTTP
-              path: /healthcheck
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            successThreshold: 1
-            timeoutSeconds: 3
+
           resources:
             limits:
               cpu: "500m"
@@ -333,36 +275,7 @@ spec:
           env:
             - name: DONGTAI_CONCURRENCY
               value: --concurrency=2
-          livenessProbe:
-            httpGet:
-              port: 5555
-              scheme: HTTP
-              path: /healthcheck
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            successThreshold: 1
-            timeoutSeconds: 3
-          readinessProbe:
-            httpGet:
-              port: 5555
-              scheme: HTTP
-              path: /healthcheck
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            successThreshold: 1
-            timeoutSeconds: 3
-          startupProbe:
-            httpGet:
-              port: 5555
-              scheme: HTTP
-              path: /healthcheck
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            successThreshold: 1
-            timeoutSeconds: 3
+
           resources:
             limits:
               cpu: "500m"
@@ -412,36 +325,7 @@ spec:
           env:
             - name: DONGTAI_CONCURRENCY
               value: --concurrency=2
-          livenessProbe:
-            httpGet:
-              port: 5555
-              scheme: HTTP
-              path: /healthcheck
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            successThreshold: 1
-            timeoutSeconds: 3
-          readinessProbe:
-            httpGet:
-              port: 5555
-              scheme: HTTP
-              path: /healthcheck
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            successThreshold: 1
-            timeoutSeconds: 3
-          startupProbe:
-            httpGet:
-              port: 5555
-              scheme: HTTP
-              path: /healthcheck
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            successThreshold: 1
-            timeoutSeconds: 3
+
           resources:
             limits:
               cpu: "500m"
@@ -491,36 +375,7 @@ spec:
           env:
             - name: DONGTAI_CONCURRENCYW
               value: -P gevent --concurrency=10
-          livenessProbe:
-            httpGet:
-              port: 5555
-              scheme: HTTP
-              path: /healthcheck
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            successThreshold: 1
-            timeoutSeconds: 3
-          readinessProbe:
-            httpGet:
-              port: 5555
-              scheme: HTTP
-              path: /healthcheck
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            successThreshold: 1
-            timeoutSeconds: 3
-          startupProbe:
-            httpGet:
-              port: 5555
-              scheme: HTTP
-              path: /healthcheck
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            successThreshold: 1
-            timeoutSeconds: 3
+
           resources:
             limits:
               cpu: "500m"
@@ -570,36 +425,7 @@ spec:
           env:
             - name: DONGTAI_CONCURRENCY
               value: -P gevent --concurrency=128
-          livenessProbe:
-            httpGet:
-              port: 5555
-              scheme: HTTP
-              path: /healthcheck
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            successThreshold: 1
-            timeoutSeconds: 3
-          readinessProbe:
-            httpGet:
-              port: 5555
-              scheme: HTTP
-              path: /healthcheck
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            successThreshold: 1
-            timeoutSeconds: 3
-          startupProbe:
-            httpGet:
-              port: 5555
-              scheme: HTTP
-              path: /healthcheck
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            successThreshold: 1
-            timeoutSeconds: 3
+
           resources:
             limits:
               cpu: "500m"
diff --git a/deploy/docker/entrypoint.sh b/deploy/docker/entrypoint.sh
index f021ab892..014369117 100755
--- a/deploy/docker/entrypoint.sh
+++ b/deploy/docker/entrypoint.sh
@@ -7,19 +7,19 @@ python -c "import deploy.docker.version_update" || true
 echo $1
 
 if [ "$1" = "worker" ]; then
-	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py worker -l info $DONGTAI_CONCURRENCY -E --pidfile=
+	celery -A dongtai_conf worker -l info $DONGTAI_CONCURRENCY -E --pidfile=
 elif [ "$1" = "worker-beat" ]; then
-	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py worker -l info -Q dongtai-periodic-task $DONGTAI_CONCURRENCY -E --pidfile=
+	celery -A dongtai_conf worker -l info -Q dongtai-periodic-task $DONGTAI_CONCURRENCY -E --pidfile=
 elif [ "$1" = "worker-high-freq" ]; then
-	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py worker -l info -Q dongtai-method-pool-scan,dongtai-replay-vul-scan $DONGTAI_CONCURRENCY -E --pidfile=
+	celery -A dongtai_conf worker -l info -Q dongtai-method-pool-scan,dongtai-replay-vul-scan $DONGTAI_CONCURRENCY -E --pidfile=
 elif [ "$1" = "worker-es" ]; then
-	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py worker -l info -Q dongtai-es-save-task $DONGTAI_CONCURRENCY -E --pidfile=
+	celery -A dongtai_conf worker -l info -Q dongtai-es-save-task $DONGTAI_CONCURRENCY -E --pidfile=
 elif [ "$1" = "worker-sca" ]; then
-	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py worker -l info -Q dongtai-sca-task,dongtai-api-route-handler $DONGTAI_CONCURRENCY -E --pidfile=
+	celery -A dongtai_conf worker -l info -Q dongtai-sca-task,dongtai-api-route-handler $DONGTAI_CONCURRENCY -E --pidfile=
 elif [ "$1" = "worker-other" ]; then
-	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py worker -l info -X dongtai-periodic-task,dongtai-method-pool-scan,dongtai-replay-vul-scan,dongtai-sca-task $DONGTAI_CONCURRENCY -E --pidfile=
+	celery -A dongtai_conf worker -l info -X dongtai-periodic-task,dongtai-method-pool-scan,dongtai-replay-vul-scan,dongtai-sca-task $DONGTAI_CONCURRENCY -E --pidfile=
 elif [ "$1" = "beat" ]; then
-	celery -A dongtai_conf flower --conf=dongtai_conf/settings.py beat -l info $DONGTAI_CONCURRENCY  --pidfile= --scheduler django_celery_beat.schedulers:DatabaseScheduler
+	celery -A dongtai_conf beat -l info $DONGTAI_CONCURRENCY  --pidfile= --scheduler django_celery_beat.schedulers:DatabaseScheduler
 else
 	echo "Get the latest vulnerability rules." && python manage.py load_hook_strategy
 	if [ $? -ne 0 ]; then echo "ERROR: Lost connection to MySQL server !!!" && exit 1 ; else echo "succeed" ;fi

From 975fa340df0d8c512219520ecf9562c7195ba8a5 Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Tue, 11 Jul 2023 18:54:32 +0800
Subject: [PATCH 109/161] feat: project version timestamp update.

---
 dongtai_web/views/project_summary.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/dongtai_web/views/project_summary.py b/dongtai_web/views/project_summary.py
index 713e8340d..11fa05173 100644
--- a/dongtai_web/views/project_summary.py
+++ b/dongtai_web/views/project_summary.py
@@ -70,6 +70,8 @@ class _ProjectSummaryDataSerializer(serializers.Serializer):
         many=True,
         help_text=_(
             "Statistics on the number of danger levels of vulnerabilities"))
+    agent_alive = serializers.IntegerField(help_text="Agent存活数量")
+    project_version_latest_time = serializers.IntegerField(help_text="项目版本更新时间")
 
 
 _ProjectSummaryResponseSerializer = get_response_serializer(

From e5d60484fe5980d6a06d585071cdafb52c622d5c Mon Sep 17 00:00:00 2001
From: tscuite 
Date: Wed, 12 Jul 2023 11:47:01 +0800
Subject: [PATCH 110/161] feat: add healthcheck

---
 .github/deploy/deploy-dongtai-server-test.yml | 25 ++++++++++++++++++-
 deploy/docker/entrypoint.sh                   |  2 ++
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/.github/deploy/deploy-dongtai-server-test.yml b/.github/deploy/deploy-dongtai-server-test.yml
index 8425ab68d..46de7eebb 100644
--- a/.github/deploy/deploy-dongtai-server-test.yml
+++ b/.github/deploy/deploy-dongtai-server-test.yml
@@ -375,7 +375,30 @@ spec:
           env:
             - name: DONGTAI_CONCURRENCYW
               value: -P gevent --concurrency=10
-
+          livenessProbe:
+            exec:
+              command: ["/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh","healthcheck"]
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            successThreshold: 1
+            timeoutSeconds: 3
+          readinessProbe:
+            exec:
+              command: ["/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh","healthcheck"]
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            successThreshold: 1
+            timeoutSeconds: 3
+          startupProbe:
+            exec:
+              command: ["/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh","healthcheck"]
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            successThreshold: 1
+            timeoutSeconds: 3
           resources:
             limits:
               cpu: "500m"
diff --git a/deploy/docker/entrypoint.sh b/deploy/docker/entrypoint.sh
index 014369117..b566347aa 100755
--- a/deploy/docker/entrypoint.sh
+++ b/deploy/docker/entrypoint.sh
@@ -20,6 +20,8 @@ elif [ "$1" = "worker-other" ]; then
 	celery -A dongtai_conf worker -l info -X dongtai-periodic-task,dongtai-method-pool-scan,dongtai-replay-vul-scan,dongtai-sca-task $DONGTAI_CONCURRENCY -E --pidfile=
 elif [ "$1" = "beat" ]; then
 	celery -A dongtai_conf beat -l info $DONGTAI_CONCURRENCY  --pidfile= --scheduler django_celery_beat.schedulers:DatabaseScheduler
+elif [ "$1" = "healthcheck" ]; then
+  celery -A dongtai_conf inspect ping -d celery@$(hostname)
 else
 	echo "Get the latest vulnerability rules." && python manage.py load_hook_strategy
 	if [ $? -ne 0 ]; then echo "ERROR: Lost connection to MySQL server !!!" && exit 1 ; else echo "succeed" ;fi

From fc3fb71122380050113c8d4d982d8fb236f55d4e Mon Sep 17 00:00:00 2001
From: st1020 
Date: Wed, 12 Jul 2023 15:53:55 +0800
Subject: [PATCH 111/161] pref: improve projects performance

---
 dongtai_web/views/projects.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dongtai_web/views/projects.py b/dongtai_web/views/projects.py
index b734d2923..e9af2475a 100644
--- a/dongtai_web/views/projects.py
+++ b/dongtai_web/views/projects.py
@@ -87,6 +87,7 @@ def get(self, request):
             queryset = queryset.filter(name__icontains=name)
         if status is not None:
             queryset = queryset.filter(status=status)
+        queryset = queryset.select_related("user")
         page_summary, page_data = self.get_paginator(queryset, page, page_size)
         vul_levels_dict = get_vul_levels_dict(
             page_data, exclude_vul_status=exclude_vul_status

From 702aedc7534353c74a0f3c14b3f40ec58236a04b Mon Sep 17 00:00:00 2001
From: tscuite 
Date: Wed, 12 Jul 2023 16:54:43 +0800
Subject: [PATCH 112/161] feat: update helm

---
 deploy/kubernetes/helm/templates/_helpers.tpl | 36 ++++++++++
 .../{dongtai-cm.yml => dongtai-cm.yaml}       |  0
 .../configmaps/dongtai-logstash.yaml          | 70 +++++++++++++++++++
 .../helm/templates/data/dongtai-mysql.yaml    | 70 +++++++++++++++++--
 .../helm/templates/data/dongtai-redis.yaml    | 15 +++-
 .../deployments/dongtai-logtsash.yaml         | 41 ++++++++++-
 .../templates/deployments/dongtai-server.yaml | 39 ++++++++++-
 .../templates/deployments/dongtai-web.yaml    |  6 +-
 .../deployments/dongtai-worker-beat.yaml      |  7 +-
 .../deployments/dongtai-worker-chain.yaml     |  7 +-
 .../deployments/dongtai-worker-es.yaml        |  7 +-
 .../deployments/dongtai-worker-high-freq.yaml |  7 +-
 .../deployments/dongtai-worker-other.yaml     |  7 +-
 .../dongtai-worker-report-only.yaml           |  7 +-
 .../deployments/dongtai-worker-sca.yaml       |  7 +-
 .../deployments/dongtai-worker-task.yaml      |  5 ++
 deploy/kubernetes/helm/values.yaml            | 27 +++++--
 17 files changed, 333 insertions(+), 25 deletions(-)
 rename deploy/kubernetes/helm/templates/configmaps/{dongtai-cm.yml => dongtai-cm.yaml} (100%)
 create mode 100644 deploy/kubernetes/helm/templates/configmaps/dongtai-logstash.yaml

diff --git a/deploy/kubernetes/helm/templates/_helpers.tpl b/deploy/kubernetes/helm/templates/_helpers.tpl
index 43061ffad..d4b86bf85 100644
--- a/deploy/kubernetes/helm/templates/_helpers.tpl
+++ b/deploy/kubernetes/helm/templates/_helpers.tpl
@@ -143,6 +143,41 @@ initContainers:
       privileged: true
 {{- end -}}
 
+{{- define "deploy.Probe" -}}
+readinessProbe:
+  exec:
+    command:
+    - bash
+    - -c
+    - celery -A dongtai_conf inspect ping -d celery@$(hostname)
+  failureThreshold: 3
+  initialDelaySeconds: 30
+  periodSeconds: 5
+  successThreshold: 1
+  timeoutSeconds: 3
+livenessProbe:
+  exec:
+    command:
+    - bash
+    - -c
+    - celery -A dongtai_conf inspect ping -d celery@$(hostname)
+  failureThreshold: 3
+  initialDelaySeconds: 30
+  periodSeconds: 5
+  successThreshold: 1
+  timeoutSeconds: 3
+startupProbe:
+  exec:
+    command:
+    - bash
+    - -c
+    - celery -A dongtai_conf inspect ping -d celery@$(hostname)
+  failureThreshold: 3
+  initialDelaySeconds: 30
+  periodSeconds: 5
+  successThreshold: 1
+  timeoutSeconds: 3
+{{- end -}}
 
 {{- define "deploy.resources" -}}
 resources:
@@ -153,6 +188,7 @@ resources:
     cpu: {{.Values.cpu}}
     memory: {{.Values.memory}}
 {{- end -}}
+
 {{- define "deploy.config.vo" -}}
 volumes:
   - name: {{ template "dongtai.fullname" . }}-configfile
diff --git a/deploy/kubernetes/helm/templates/configmaps/dongtai-cm.yml b/deploy/kubernetes/helm/templates/configmaps/dongtai-cm.yaml
similarity index 100%
rename from deploy/kubernetes/helm/templates/configmaps/dongtai-cm.yml
rename to deploy/kubernetes/helm/templates/configmaps/dongtai-cm.yaml
diff --git a/deploy/kubernetes/helm/templates/configmaps/dongtai-logstash.yaml b/deploy/kubernetes/helm/templates/configmaps/dongtai-logstash.yaml
new file mode 100644
index 000000000..8ea794012
--- /dev/null
+++ b/deploy/kubernetes/helm/templates/configmaps/dongtai-logstash.yaml
@@ -0,0 +1,70 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: agent-config
+  namespace: {{ .Release.Namespace }}
+data:
+  logstash.conf: |
+    input {
+      http {
+        host => "0.0.0.0"
+        port => 8082
+        user => "sjkadfkjdsahfkds"
+        password => "dsjkfkjsadfhkjsadhfksdsjdafjkdsagfjhsadgfjsdfg12312!"
+        type => "tcp"
+        codec => "json"
+      }
+      tcp {
+        port => 8083
+        type => "log"
+      }
+    }
+    filter {
+      if [type] == "tcp"{
+        ruby {
+           code => "event.set('timestamp', event.get('@timestamp').time.localtime + 8*60*60)"
+        }
+        ruby {
+           code => "event.set('@timestamp',event.get('timestamp'))"
+        }
+        mutate {
+           remove_field => ["timestamp","@version","host","date","headers"]
+        }
+        # grok {
+        #   match => {
+        #     "log" => ".*\[%{DATA:agent_log_level}\].*"
+        #   }
+        # }
+      }
+      if [type] == "log"{
+        json{
+          source => ["message"]
+          remove_field => ["message"]
+        }
+      }
+    }
+    output {
+      if [type] == "tcp"{
+        file {
+            path => ["/tmp/logstash/agent/%{[agent]}/%{+YYYY.MM.dd}.log"]
+        }
+        # elasticsearch {
+        #   hosts => ["10.12.9.20:9200"]
+        #   index => "agent-log-%{+YYYY.MM.dd}"
+        # }
+      }
+      if [type] == "log"{
+        #stdout { 
+        #  codec => rubydebug
+        #}
+        analyticdb {
+            driver_class => "com.mysql.jdbc.Driver"
+            max_pool_size => "100"
+            connection_test_query => "SELECT 1"
+            connection_string => "jdbc:mysql://${DATABASE}?useSSL=false"
+            username => "${USERNAME}"
+            password => "${PASSWORD}"
+            statement => [ "INSERT IGNORE INTO `iast_agent_method_pool` (`agent_id`,`url`, `uri`, `http_method`, `http_scheme`, `http_protocol`, `req_header`, `req_params`, `req_data`, `res_header`, `res_body`, `context_path`, `method_pool`, `clent_ip`, `create_time`, `update_time`, `pool_sign`, `req_header_for_search`, `uri_sha1`) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)","agent_id","url","uri","http_method","http_scheme","http_protocol","req_header","req_params","req_data","res_header","res_body","context_path","method_pool","clent_ip","create_time","update_time","pool_sign","req_header_for_search","uri_sha1"]
+        }
+      }
+    }
\ No newline at end of file
diff --git a/deploy/kubernetes/helm/templates/data/dongtai-mysql.yaml b/deploy/kubernetes/helm/templates/data/dongtai-mysql.yaml
index 77d571756..e014e561a 100644
--- a/deploy/kubernetes/helm/templates/data/dongtai-mysql.yaml
+++ b/deploy/kubernetes/helm/templates/data/dongtai-mysql.yaml
@@ -16,7 +16,11 @@ spec:
   template:
     metadata:
       annotations:
-        build_number: "{{ template "dongtai.fullname" . }}"
+        {{- if .Values.develop.dev }}
+        sidecar.istio.io/inject: "true"
+        sidecar.istio.io/interceptionMode: TPROXY
+        {{- end }}
+        mysql_number: {{.Values.build.mysql_number}}
         {{- if not .Values.skipistio }}{{ include "dongtai.istiolabels" . | nindent 8 }}{{ end }}
       labels:
         app: {{ template "dongtai.fullname" . }}-mysql
@@ -28,10 +32,62 @@ spec:
 {{ toYaml .Values.nodeSelector | indent 8 }}
 {{- end }}
       containers:
-        - image: {{ .Values.images }}/dongtai-mysql:{{ .Values.tag }}
-          name: mysql-container
-          imagePullPolicy: Always
-          ports:
-            - containerPort: 3306
-              name: tcp-mysql
+      - image: {{ .Values.images }}/dongtai-mysql:{{ .Values.tag }}
+        name: mysql-container
+        imagePullPolicy: Always
+        {{- if .Values.develop.dev }}
+        lifecycle:
+          postStart:
+            exec:
+              command:
+              - /bin/bash
+              - -c
+              - {{.Values.develop.mysqlLifecycle}}
+        {{- end }}
+        ports:
+          - containerPort: 3306
+            name: tcp-mysql
+        {{- if .Values.healthcheck }}
+        livenessProbe:
+          failureThreshold: 3
+          initialDelaySeconds: 30
+          periodSeconds: 5
+          successThreshold: 1
+          tcpSocket:
+            port: 3306
+          timeoutSeconds: 1
+        readinessProbe:
+          failureThreshold: 3
+          initialDelaySeconds: 30
+          periodSeconds: 5
+          successThreshold: 1
+          tcpSocket:
+            port: 3306
+          timeoutSeconds: 1
+        startupProbe:
+          failureThreshold: 40
+          periodSeconds: 5
+          successThreshold: 1
+          tcpSocket:
+            port: 3306
+          timeoutSeconds: 1
+        {{- end }}
+        {{- if .Values.develop.dev }}
+        resources:
+          limits:
+            cpu: {{.Values.develop.mysqlCpu}}
+            memory: {{.Values.develop.mysqlMemory}}
+          requests:
+            cpu: {{.Values.develop.mysqlCpu}}
+            memory: {{.Values.develop.mysqlMemory}}
+        volumeMounts:
+        - mountPath: /tmp/poststart.sql
+          name: hookvolume
+          subPath: poststart.sql
+      volumes:
+      - configMap:
+          defaultMode: 493
+          name: poststarthook
+        name: hookvolume
+        {{- end }}
 {{- end -}}
\ No newline at end of file
diff --git a/deploy/kubernetes/helm/templates/data/dongtai-redis.yaml b/deploy/kubernetes/helm/templates/data/dongtai-redis.yaml
index d2bedb486..c38798d9e 100644
--- a/deploy/kubernetes/helm/templates/data/dongtai-redis.yaml
+++ b/deploy/kubernetes/helm/templates/data/dongtai-redis.yaml
@@ -14,7 +14,11 @@ spec:
   template:
     metadata:
       annotations:
-        build_number: "{{ template "dongtai.fullname" . }}"
+        {{- if .Values.develop.dev }}
+        sidecar.istio.io/inject: "true"
+        sidecar.istio.io/interceptionMode: TPROXY
+        {{- end }}
+        redis_number: {{.Values.build.redis_number}}
         {{- if not .Values.skipistio }}{{ include "dongtai.istiolabels" . | nindent 8 }}{{ end }}
       labels:
         app: {{ template "dongtai.fullname" . }}-redis
@@ -33,4 +37,13 @@ spec:
         - containerPort: 6379
           name: redis
           protocol: TCP
+        {{- if .Values.develop.dev }}
+        resources:
+          limits:
+            cpu: {{.Values.develop.redisCpu}}
+            memory: {{.Values.develop.redisMemory}}
+          requests:
+            cpu: {{.Values.develop.redisCpu}}
+            memory: {{.Values.develop.redisMemory}}
+        {{- end }}
 {{- end -}}
\ No newline at end of file
diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-logtsash.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-logtsash.yaml
index ece082964..320f49c15 100644
--- a/deploy/kubernetes/helm/templates/deployments/dongtai-logtsash.yaml
+++ b/deploy/kubernetes/helm/templates/deployments/dongtai-logtsash.yaml
@@ -16,6 +16,10 @@ spec:
   template:
     metadata:
       annotations:
+        {{- if .Values.develop.dev }}
+        sidecar.istio.io/inject: "true"
+        sidecar.istio.io/interceptionMode: TPROXY
+        {{- end }}
         build_number: "{{ template "dongtai.fullname" . }}"
         {{- if not .Values.skipistio }}{{ include "dongtai.istiolabels" . | nindent 8}}{{ end }}
       labels:
@@ -55,6 +59,30 @@ spec:
         - containerPort: 8083
           protocol: TCP
           name: log-http
+        {{- if .Values.healthcheck }}
+        livenessProbe:
+          failureThreshold: 1
+          periodSeconds: 5
+          successThreshold: 1
+          tcpSocket:
+            port: 9600
+          timeoutSeconds: 1
+        readinessProbe:
+          failureThreshold: 3
+          initialDelaySeconds: 30
+          periodSeconds: 5
+          successThreshold: 1
+          tcpSocket:
+            port: 9600
+          timeoutSeconds: 1
+        startupProbe:
+          failureThreshold: 40
+          periodSeconds: 5
+          successThreshold: 1
+          tcpSocket:
+            port: 9600
+          timeoutSeconds: 1
+        {{- end }}
         resources:
           requests:
             cpu: 1000m
@@ -62,15 +90,24 @@ spec:
           limits:
             cpu: 2000m
             memory: 4000Mi
-{{- if .Values.storage.persistentVolumeClaim }}
         volumeMounts:
+        - mountPath: /usr/share/logstash/pipeline/logstash.conf
+          name: agent-config
+          subPath: logstash.conf
+{{- if .Values.storage.persistentVolumeClaim }}
         - name: {{ template "dongtai.fullname" . }}-log-path
           mountPath: /tmp/logstash
+{{- end }}
       volumes:
+      - configMap:
+          defaultMode: 420
+          name: agent-config
+        name: agent-config
+{{- if .Values.storage.persistentVolumeClaim }}
       - name: {{ template "dongtai.fullname" . }}-log-path
         persistentVolumeClaim:
           {{ include "deploy.config.persistentVolumeClaim" . }}
-{{ end -}}
+{{- end }}
       {{- if .Values.somaxconn }}
       {{- include "deploy.initContainers" . | nindent 6 }}
       {{- end }}
\ No newline at end of file
diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-server.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-server.yaml
index ca3e147cf..420bbbc20 100644
--- a/deploy/kubernetes/helm/templates/deployments/dongtai-server.yaml
+++ b/deploy/kubernetes/helm/templates/deployments/dongtai-server.yaml
@@ -20,6 +20,8 @@ spec:
     metadata:
       annotations:
         build_number: "{{ template "dongtai.fullname" . }}"
+        agent_number: {{.Values.build.agent_number}}
+        server_number: {{.Values.build.server_number}}
         {{- if not .Values.skipistio }}{{ include "dongtai.istiolabels" . | nindent 8 }}{{ end }}
       labels:
         app: {{ template "dongtai.fullname" . }}-server
@@ -34,12 +36,45 @@ spec:
           image: {{ .Values.images }}/dongtai-server:{{ .Values.tag }}
           env:
             - name: DONGTAI_CONCURRENCY
-              value: --processes 4
-            {{- range $key, $value :=  .Values.env }}
+              value: {{.Values.build.env_server}}
+            {{- range $key, $value :=  .Values.serverEnv }}
             - name: {{ $key }}
               value: {{ $value | quote }}
             {{- end }}
           {{- include "deploy.config" . | nindent 10 }}
+          {{- if .Values.healthcheck }}
+          livenessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /healthcheck
+              port: 8000
+              scheme: HTTP
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 3
+          name: dongtai-server-container
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /healthcheck
+              port: 8000
+              scheme: HTTP
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 3
+          startupProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /healthcheck
+              port: 8000
+              scheme: HTTP
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 3
+          {{- end }}
       {{- include "deploy.config.vo" . | nindent 6 }}
       {{- if .Values.somaxconn }}
       {{- include "deploy.initContainers" . | nindent 6 }}
diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-web.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-web.yaml
index eda74f024..71d2013c6 100644
--- a/deploy/kubernetes/helm/templates/deployments/dongtai-web.yaml
+++ b/deploy/kubernetes/helm/templates/deployments/dongtai-web.yaml
@@ -19,7 +19,11 @@ spec:
   template:
     metadata:
       annotations:
-        build_number: "{{ template "dongtai.fullname" . }}"
+        {{- if .Values.develop.dev }}
+        sidecar.istio.io/inject: "true"
+        sidecar.istio.io/interceptionMode: TPROXY
+        {{- end }}
+        web_number: {{.Values.build.web_number}}
         {{- if not .Values.skipistio }}{{ include "dongtai.istiolabels" . | nindent 8 }}{{ end }}
       labels:
         app: {{ template "dongtai.fullname" . }}-web
diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-beat.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-beat.yaml
index 650162601..d74c71454 100644
--- a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-beat.yaml
+++ b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-beat.yaml
@@ -18,6 +18,8 @@ spec:
       {{- include "dongtai.labels" .  | nindent 6 }}
   template:
     metadata:
+      annotations:
+        server_number: {{.Values.build.server_number}}
       labels:
         app: {{ template "dongtai.fullname" . }}-worker-beat
         {{- include "dongtai.labels" .  | nindent 8 }}
@@ -33,6 +35,9 @@ spec:
           args: [ "worker-beat" ]
           env:
             - name: DONGTAI_CONCURRENCY
-              value: --concurrency=2
+              value: {{.Values.build.env_beta}}
           {{- include "deploy.config" . | nindent 10 }}
+          {{- if .Values.healthcheck }}
+          {{- include "deploy.Probe" . | nindent 10 }}
+          {{- end }}
       {{- include "deploy.config.vo" . | nindent 6 }}
\ No newline at end of file
diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-chain.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-chain.yaml
index 9fbe8a419..7b9111c5b 100644
--- a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-chain.yaml
+++ b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-chain.yaml
@@ -19,6 +19,8 @@ spec:
       {{- include "dongtai.labels" .  | nindent 6 }}
   template:
     metadata:
+      annotations:
+        server_number: {{.Values.build.server_number}}
       labels:
         app: {{ template "dongtai.fullname" . }}-worker-chain
         {{- include "dongtai.labels" .  | nindent 8 }}
@@ -34,7 +36,10 @@ spec:
           args: [ "worker-chain" ]
           env:
             - name: DONGTAI_CONCURRENCY
-              value: --concurrency=12
+              value: {{.Values.build.env_chain}}
           {{- include "deploy.config" . | nindent 10 }}
+          {{- if .Values.healthcheck }}
+          {{- include "deploy.Probe" . | nindent 10 }}
+          {{- end }}
       {{- include "deploy.config.vo" . | nindent 6 }}
 {{- end }}
\ No newline at end of file
diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-es.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-es.yaml
index fe801fdd0..d7ac958fc 100644
--- a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-es.yaml
+++ b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-es.yaml
@@ -18,6 +18,8 @@ spec:
       {{- include "dongtai.labels" .  | nindent 6 }}
   template:
     metadata:
+      annotations:
+        server_number: {{.Values.build.server_number}}
       labels:
         app: {{ template "dongtai.fullname" . }}-worker-es
         {{- include "dongtai.labels" .  | nindent 8 }}
@@ -33,6 +35,9 @@ spec:
           args: [ "worker-es" ]
           env:
             - name: DONGTAI_CONCURRENCY
-              value: -P gevent --concurrency=64
+              value: {{.Values.build.env_es}}
           {{- include "deploy.config" . | nindent 10 }}
+          {{- if .Values.healthcheck }}
+          {{- include "deploy.Probe" . | nindent 10 }}
+          {{- end }}
       {{- include "deploy.config.vo" . | nindent 6 }}
\ No newline at end of file
diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-high-freq.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-high-freq.yaml
index 7f90df696..0b585ae40 100644
--- a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-high-freq.yaml
+++ b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-high-freq.yaml
@@ -18,6 +18,8 @@ spec:
       {{- include "dongtai.labels" .  | nindent 6 }}
   template:
     metadata:
+      annotations:
+        server_number: {{.Values.build.server_number}}
       labels:
         app: {{ template "dongtai.fullname" . }}-worker-high-freq
         {{- include "dongtai.labels" .  | nindent 8 }}
@@ -33,6 +35,9 @@ spec:
           args: [ "worker-high-freq" ]
           env:
             - name: DONGTAI_CONCURRENCY
-              value: -P gevent --concurrency=121
+              value: {{.Values.build.env_high}}
           {{- include "deploy.config" . | nindent 10 }}
+          {{- if .Values.healthcheck }}
+          {{- include "deploy.Probe" . | nindent 10 }}
+          {{- end }}
       {{- include "deploy.config.vo" . | nindent 6 }}
\ No newline at end of file
diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-other.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-other.yaml
index b53d744ab..28f2b7cb0 100644
--- a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-other.yaml
+++ b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-other.yaml
@@ -18,6 +18,8 @@ spec:
       {{- include "dongtai.labels" .  | nindent 6 }}
   template:
     metadata:
+      annotations:
+        server_number: {{.Values.build.server_number}}
       labels:
         app: {{ template "dongtai.fullname" . }}-worker-other
         {{- include "dongtai.labels" .  | nindent 8 }}
@@ -33,6 +35,9 @@ spec:
           args: [ "worker-other" ]
           env:
             - name: DONGTAI_CONCURRENCY
-              value: --concurrency=10
+              value: {{.Values.build.env_other}}
           {{- include "deploy.config" . | nindent 10 }}
+          {{- if .Values.healthcheck }}
+          {{- include "deploy.Probe" . | nindent 10 }}
+          {{- end }}
       {{- include "deploy.config.vo" . | nindent 6 }}
\ No newline at end of file
diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-report-only.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-report-only.yaml
index e9c98cf6b..fd4d970fd 100644
--- a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-report-only.yaml
+++ b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-report-only.yaml
@@ -19,6 +19,8 @@ spec:
       {{- include "dongtai.labels" .  | nindent 6 }}
   template:
     metadata:
+      annotations:
+        server_number: {{.Values.build.server_number}}
       labels:
         app: {{ template "dongtai.fullname" . }}-worker-report-only
         {{- include "dongtai.labels" .  | nindent 8 }}
@@ -32,6 +34,9 @@ spec:
           image: {{ .Values.images }}/dongtai-server:{{ .Values.tag }}
           command: [ "/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh" ]
           args: [ "worker-report-only" ]
-          {{- include "deploy.configmax" . | nindent 10 }}
+          {{- include "deploy.config" . | nindent 10 }}
+          {{- if .Values.healthcheck }}
+          {{- include "deploy.Probe" . | nindent 10 }}
+          {{- end }}
       {{- include "deploy.config.vo" . | nindent 6 }}
 {{- end }}
\ No newline at end of file
diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-sca.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-sca.yaml
index 17bfdb00b..07f5b9a21 100644
--- a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-sca.yaml
+++ b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-sca.yaml
@@ -18,6 +18,8 @@ spec:
       {{- include "dongtai.labels" .  | nindent 6 }}
   template:
     metadata:
+      annotations:
+        server_number: {{.Values.build.server_number}}
       labels:
         app: {{ template "dongtai.fullname" . }}-worker-sca
         {{- include "dongtai.labels" .  | nindent 8 }}
@@ -33,6 +35,9 @@ spec:
           args: [ "worker-sca" ]
           env:
             - name: DONGTAI_CONCURRENCY
-              value: -P gevent --concurrency=10
+              value: {{.Values.build.env_server}}
           {{- include "deploy.config" . | nindent 10 }}
+          {{- if .Values.healthcheck }}
+          {{- include "deploy.Probe" . | nindent 10 }}
+          {{- end }}
       {{- include "deploy.config.vo" . | nindent 6 }}
\ No newline at end of file
diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-task.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-task.yaml
index 0669c81d7..e43238e84 100644
--- a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-task.yaml
+++ b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-task.yaml
@@ -18,6 +18,8 @@ spec:
       {{- include "dongtai.labels" .  | nindent 6 }}
   template:
     metadata:
+      annotations:
+        server_number: {{.Values.build.server_number}}
       labels:
         app: {{ template "dongtai.fullname" . }}-worker-task
         {{- include "dongtai.labels" .  | nindent 8 }}
@@ -32,4 +34,7 @@ spec:
           command: [ "/bin/bash","/opt/dongtai/deploy/docker/entrypoint.sh" ]
           args: [ "beat" ]
           {{- include "deploy.config" . | nindent 10 }}
+          # {{- if .Values.healthcheck }}
+          # {{- include "deploy.Probe" . | nindent 10 }}
+          # {{- end }}
       {{- include "deploy.config.vo" . | nindent 6 }}
\ No newline at end of file
diff --git a/deploy/kubernetes/helm/values.yaml b/deploy/kubernetes/helm/values.yaml
index 4a269958c..aa3347823 100644
--- a/deploy/kubernetes/helm/values.yaml
+++ b/deploy/kubernetes/helm/values.yaml
@@ -15,15 +15,14 @@ accessType: ClusterIP  # NodePort, LoadBalancer, ClusterIP
 NodePort: 30080
 logging_level: INFO # DEBUG, INFO
 somaxconn: null #If system max net.core.somaxconn (128) . Example: somaxconn: 4096
+healthcheck: false
 
 nodeSelector:
   kubernetes.io/os: linux
   kubernetes.io/arch: amd64
 podLabels: {}
   # example: "test"
-env: {}
-  # example: iast-foo
-  
+
 skipMysql: false
 mysql:
   host: dongtai-mysql
@@ -44,7 +43,7 @@ sca:
   sca_token:
 
 usb:
-  usb_token: 837e62834b8423418c2a2axxxxxxxxxxxxxxx
+  usb_token:
 
 storage:
   storageClassName: null
@@ -60,4 +59,22 @@ smtp:
   password: smtp_password
   from_addr: from_addr
   ssl: False
-  cc_addr: cc_addr
\ No newline at end of file
+  cc_addr: cc_addr
+
+build:
+  agent_number: agent_number_version
+  web_number: agent_number_version
+  server_number: agent_number_version
+  logstash_number: agent_number_version
+  mysql_number: agent_number_version
+  redis_number: agent_number_version
+  env_server: --processes 4
+  env_beta: --concurrency=2
+  env_chain: --concurrency=12
+  env_es: -P gevent --concurrency=64
+  env_high: -P gevent --concurrency=121
+  env_other: --concurrency=10
+  env_sca: -P gevent --concurrency=10
+
+develop:
+  dev:
\ No newline at end of file

From fa92c6a984123aac9d4f420de81622df077b7229 Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Wed, 12 Jul 2023 17:28:26 +0800
Subject: [PATCH 113/161] fix: celery connection error
 https://github.com/celery/celery/pull/7951 .

---
 dongtai_conf/celery.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dongtai_conf/celery.py b/dongtai_conf/celery.py
index 58fa30790..22b2ad042 100644
--- a/dongtai_conf/celery.py
+++ b/dongtai_conf/celery.py
@@ -61,7 +61,7 @@
 configs['task_always_eager'] = True if os.getenv(
     "CELERY_EAGER_TEST") == "TRUE" else False
 configs['task_acks_on_failure_or_timeout'] = True
-configs['broker_connection_retry_on_startup'] = False
+configs['broker_channel_error_retry'] = True
 configs['broker_connection_max_retries'] = 0  # it means retry forever
 configs[
     'broker_pool_limit'] = 1000  # to forbid contention can arise when using gevent.

From 861d88af6a84bc3e65866968d9a28f5521e8d7cb Mon Sep 17 00:00:00 2001
From: tscuite 
Date: Wed, 12 Jul 2023 17:45:34 +0800
Subject: [PATCH 114/161] feat: update ci

---
 .github/deploy/deploy-dongtai-server-test.yml | 464 ------------------
 .github/deploy/devops.yml                     |  54 ++
 .github/workflows/deploy-dev.yaml             |  37 +-
 .../service/dongtai-logstash-svc.yaml         |  16 +-
 .../helm/templates/service/dongtai-mysql.yaml |   7 +-
 .../helm/templates/service/dongtai-redis.yaml |   7 +-
 .../templates/service/dongtai-server-svc.yaml |   3 +-
 .../templates/service/dongtai-web-svc.yaml    |   3 +-
 8 files changed, 88 insertions(+), 503 deletions(-)
 delete mode 100644 .github/deploy/deploy-dongtai-server-test.yml
 create mode 100644 .github/deploy/devops.yml

diff --git a/.github/deploy/deploy-dongtai-server-test.yml b/.github/deploy/deploy-dongtai-server-test.yml
deleted file mode 100644
index 46de7eebb..000000000
--- a/.github/deploy/deploy-dongtai-server-test.yml
+++ /dev/null
@@ -1,464 +0,0 @@
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: entrypoint
-  namespace: iast-test
-data:
-  entrypoint.sh: |-
-    #!/bin/bash
-    echo '启动uwsgi服务'
-    /bin/bash -c 'mkdir -p /tmp/{logstash/{batchagent,report/{img,word,pdf,excel,html}},iast_cache/package}'
-    python manage.py compilemessages  
-    sleep 2
-    python -c "import deploy.docker.version_update" || true
-    echo $1
-    
-    if [ "$1" = "worker" ]; then
-    	celery -A dongtai_conf worker -l info $DONGTAI_CONCURRENCY -E --pidfile= 
-    elif [ "$1" = "worker-beat" ]; then
-    	celery -A dongtai_conf worker -l info -Q dongtai-periodic-task $DONGTAI_CONCURRENCY -E --pidfile=
-    elif [ "$1" = "worker-high-freq" ]; then
-    	celery -A dongtai_conf worker -l info -Q dongtai-method-pool-scan,dongtai-replay-vul-scan $DONGTAI_CONCURRENCY -E --pidfile=
-    elif [ "$1" = "worker-es" ]; then
-    	celery -A dongtai_conf worker -l info -Q dongtai-es-save-task $DONGTAI_CONCURRENCY -E --pidfile=  
-    elif [ "$1" = "worker-sca" ]; then
-    	celery -A dongtai_conf worker -l info -Q dongtai-sca-task $DONGTAI_CONCURRENCY -E --pidfile=  
-    elif [ "$1" = "worker-other" ]; then
-    	celery -A dongtai_conf worker -l info -X dongtai-periodic-task,dongtai-method-pool-scan,dongtai-replay-vul-scan,dongtai-sca-task $DONGTAI_CONCURRENCY -E --pidfile= 
-    elif [ "$1" = "beat" ]; then
-    	celery -A dongtai_conf beat -l info $DONGTAI_CONCURRENCY  --pidfile= --scheduler django_celery_beat.schedulers:DatabaseScheduler
-    else
-    	# echo "Get the latest vulnerability rules." && python manage.py load_hook_strategy 2> /dev/null
-    	# if [ $? -ne 0 ]; then echo "ERROR: Lost connection to MySQL server !!!" && exit 1 ; else echo "succeed" ;fi
-    	uwsgi --ini /opt/dongtai/dongtai_conf/conf/uwsgi.ini $DONGTAI_CONCURRENCY
-    fi
----
-# dongtai-server服务
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: dongtai-server
-  namespace: iast-test
-  annotations:
-    kubesphere.io/description: dongtai-server
-  labels:
-    app: dongtai-server
-    version: v1
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: dongtai-server
-  template:
-    metadata:
-      annotations:
-        sidecar.istio.io/inject: "true"
-      labels:
-        app: dongtai-server
-        version: v1
-    spec:
-      containers:
-        - name: dongtai-server-container
-          image: registry.cn-hongkong.aliyuncs.com/secnium/dongtai-server-test:VERSION
-          imagePullPolicy: Always
-          volumeMounts:
-            - name: configfile
-              mountPath: /opt/dongtai/dongtai_conf/conf/config.ini
-              subPath: config.ini
-            - name: entrypoint
-              mountPath: /opt/dongtai/deploy/docker/entrypoint.sh
-              subPath: entrypoint.sh
-            - name: log-path
-              mountPath: /tmp/logstash
-          env:
-            - name: DONGTAI_CONCURRENCY
-              value: --processes 2 --stats :3031 --stats-http
-            - name: METRICS
-              value: =true
-          livenessProbe:
-            httpGet:
-              port: 8000
-              scheme: HTTP
-              path: /healthcheck
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            successThreshold: 1
-            timeoutSeconds: 3
-          readinessProbe:
-            httpGet:
-              port: 8000
-              scheme: HTTP
-              path: /healthcheck
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            successThreshold: 1
-            timeoutSeconds: 3
-          startupProbe:
-            httpGet:
-              port: 8000
-              scheme: HTTP
-              path: /healthcheck
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            successThreshold: 1
-            timeoutSeconds: 3
-          resources:
-            limits:
-              cpu: "500m"
-              memory: 1000Mi
-            requests:
-              cpu: "500m"
-              memory: 500Mi
-      volumes:
-        - name: configfile
-          configMap:
-            name: dongtai-iast-config.ini
-        - name: entrypoint
-          configMap:
-            name: entrypoint
-        - name: log-path
-          persistentVolumeClaim:
-            claimName: app-agent-pvc
-      imagePullSecrets:
-        - name: aliyun-registry-secret
----
-# dongtai-server服务
-apiVersion: v1
-kind: Service
-metadata:
-  name: dongtai-server-svc
-  namespace: iast-test
-spec:
-  ports:
-    - name: http
-      protocol: TCP
-      port: 80
-      targetPort: 8000
-  selector:
-    app: dongtai-server
-  type: ClusterIP
----
-#dongtai-worker-task服务
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: dongtai-worker-task
-  namespace: iast-test
-  annotations:
-    kubesphere.io/description: dongtai-worker-task
-  labels:
-    app: dongtai-worker-task
-    version: v1
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: dongtai-worker-task
-  template:
-    metadata:
-      labels:
-        app: dongtai-worker-task
-        version: v1
-    spec:
-      containers:
-        - name: dongtai-worker-task-container
-          image: registry.cn-hongkong.aliyuncs.com/secnium/dongtai-server-test:VERSION
-          command: [ "/bin/bash","/opt/dongtai/deploy/docker/entrypoint.sh" ]
-          args: [ "beat" ]
-          imagePullPolicy: Always
-          volumeMounts:
-            - name: configfile
-              mountPath: /opt/dongtai/dongtai_conf/conf/config.ini
-              subPath: config.ini
-
-          resources:
-            limits:
-              cpu: "500m"
-              memory: 1000Mi
-            requests:
-              cpu: "500m"
-              memory: 500Mi
-      volumes:
-        - name: configfile
-          configMap:
-            name: dongtai-iast-config.ini
-      imagePullSecrets:
-        - name: aliyun-registry-secret
----
-
-# dongtai-worker-high-freq服务
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: dongtai-worker-high-freq
-  namespace: iast-test
-  annotations:
-    kubesphere.io/description: dongtai-worker-high-freq
-  labels:
-    app: dongtai-worker-high-freq
-    version: v1
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: dongtai-worker-high-freq
-  template:
-    metadata:
-      labels:
-        app: dongtai-worker-high-freq
-        version: v1
-    spec:
-      containers:
-        - name: dongtai-worker-high-freq-container
-          image: registry.cn-hongkong.aliyuncs.com/secnium/dongtai-server-test:VERSION
-          command: [ "/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh" ]
-          args: [ "worker-high-freq" ]
-          imagePullPolicy: Always
-          volumeMounts:
-            - name: configfile
-              mountPath: /opt/dongtai/dongtai_conf/conf/config.ini
-              subPath: config.ini
-          env:
-            - name: DONGTAI_CONCURRENCY
-              value: -P gevent --concurrency=121
-
-          resources:
-            limits:
-              cpu: "500m"
-              memory: 1000Mi
-            requests:
-              cpu: "500m"
-              memory: 500Mi
-      volumes:
-        - name: configfile
-          configMap:
-            name: dongtai-iast-config.ini
-      imagePullSecrets:
-        - name: aliyun-registry-secret
----
-# dongtai-worker-beat服务
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: dongtai-worker-beat
-  namespace: iast-test
-  annotations:
-    kubesphere.io/description: dongtai-worker-beat
-  labels:
-    app: dongtai-worker-beat
-    version: v1
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: dongtai-worker-beat
-  template:
-    metadata:
-      labels:
-        app: dongtai-worker-beat
-        version: v1
-    spec:
-      containers:
-        - name: dongtai-worker-beat-container
-          image: registry.cn-hongkong.aliyuncs.com/secnium/dongtai-server-test:VERSION
-          command: [ "/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh" ]
-          args: [ "worker-beat" ]
-          imagePullPolicy: Always
-          volumeMounts:
-            - name: configfile
-              mountPath: /opt/dongtai/dongtai_conf/conf/config.ini
-              subPath: config.ini
-          env:
-            - name: DONGTAI_CONCURRENCY
-              value: --concurrency=2
-
-          resources:
-            limits:
-              cpu: "500m"
-              memory: 1000Mi
-            requests:
-              cpu: "500m"
-              memory: 500Mi
-      volumes:
-        - name: configfile
-          configMap:
-            name: dongtai-iast-config.ini
-      imagePullSecrets:
-        - name: aliyun-registry-secret
----
-# dongtai-worker-other服务
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: dongtai-worker-other
-  namespace: iast-test
-  annotations:
-    kubesphere.io/description: dongtai-worker-other
-  labels:
-    app: dongtai-worker-other
-    version: v1
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: dongtai-worker-other
-  template:
-    metadata:
-      labels:
-        app: dongtai-worker-other
-        version: v1
-    spec:
-      containers:
-        - name: dongtai-worker-other-container
-          image: registry.cn-hongkong.aliyuncs.com/secnium/dongtai-server-test:VERSION
-          command: [ "/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh" ]
-          args: [ "worker-other" ]
-          imagePullPolicy: Always
-          volumeMounts:
-            - name: configfile
-              mountPath: /opt/dongtai/dongtai_conf/conf/config.ini
-              subPath: config.ini
-          env:
-            - name: DONGTAI_CONCURRENCY
-              value: --concurrency=2
-
-          resources:
-            limits:
-              cpu: "500m"
-              memory: 1000Mi
-            requests:
-              cpu: "500m"
-              memory: 500Mi
-      volumes:
-        - name: configfile
-          configMap:
-            name: dongtai-iast-config.ini
-      imagePullSecrets:
-        - name: aliyun-registry-secret
----
-# dongtai-worker-sca服务
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: dongtai-worker-sca
-  namespace: iast-test
-  annotations:
-    kubesphere.io/description: dongtai-worker-sca
-  labels:
-    app: dongtai-worker-sca
-    version: v1
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: dongtai-worker-sca
-  template:
-    metadata:
-      labels:
-        app: dongtai-worker-sca
-        version: v1
-    spec:
-      containers:
-        - name: dongtai-worker-sca-container
-          image: registry.cn-hongkong.aliyuncs.com/secnium/dongtai-server-test:VERSION
-          command: [ "/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh" ]
-          args: [ "worker-sca" ]
-          imagePullPolicy: Always
-          volumeMounts:
-            - name: configfile
-              mountPath: /opt/dongtai/dongtai_conf/conf/config.ini
-              subPath: config.ini
-          env:
-            - name: DONGTAI_CONCURRENCYW
-              value: -P gevent --concurrency=10
-          livenessProbe:
-            exec:
-              command: ["/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh","healthcheck"]
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            successThreshold: 1
-            timeoutSeconds: 3
-          readinessProbe:
-            exec:
-              command: ["/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh","healthcheck"]
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            successThreshold: 1
-            timeoutSeconds: 3
-          startupProbe:
-            exec:
-              command: ["/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh","healthcheck"]
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            successThreshold: 1
-            timeoutSeconds: 3
-          resources:
-            limits:
-              cpu: "500m"
-              memory: 1000Mi
-            requests:
-              cpu: "500m"
-              memory: 500Mi
-      volumes:
-        - name: configfile
-          configMap:
-            name: dongtai-iast-config.ini
-      imagePullSecrets:
-        - name: aliyun-registry-secret
----
-# dongtai-worker-es服务
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: dongtai-worker-es
-  namespace: iast-test
-  annotations:
-    kubesphere.io/description: dongtai-worker-es
-  labels:
-    app: dongtai-worker-es
-    version: v1
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: dongtai-worker-es
-  template:
-    metadata:
-      labels:
-        app: dongtai-worker-es
-        version: v1
-    spec:
-      containers:
-        - name: dongtai-worker-es-container
-          image: registry.cn-hongkong.aliyuncs.com/secnium/dongtai-server-test:VERSION
-          command: [ "/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh" ]
-          args: [ "worker-es" ]
-          imagePullPolicy: Always
-          volumeMounts:
-            - name: configfile
-              mountPath: /opt/dongtai/dongtai_conf/conf/config.ini
-              subPath: config.ini
-          env:
-            - name: DONGTAI_CONCURRENCY
-              value: -P gevent --concurrency=128
-
-          resources:
-            limits:
-              cpu: "500m"
-              memory: 1000Mi
-            requests:
-              cpu: "500m"
-              memory: 500Mi
-      volumes:
-        - name: configfile
-          configMap:
-            name: dongtai-iast-config.ini
-      imagePullSecrets:
-        - name: aliyun-registry-secret
diff --git a/.github/deploy/devops.yml b/.github/deploy/devops.yml
new file mode 100644
index 000000000..d8eb5d3f6
--- /dev/null
+++ b/.github/deploy/devops.yml
@@ -0,0 +1,54 @@
+images: registry.cn-beijing.aliyuncs.com/huoxian_pub    # or   dongtai
+tag:
+imagePullPolicy: Always
+replicaCount: 1
+cpu: 500m
+memory: 1000Mi
+logging_level: DEBUG # DEBUG, INFO
+healthcheck: true
+
+mysql:
+  host:
+
+sca:
+  sca_token:
+
+storage:
+  storageClassName: alicloud-nas-subpath
+
+csrfTrustOrigins:
+
+build:
+  agent_number: agent_number_version
+  web_number: agent_number_version
+  server_number: agent_number_version
+  logstash_number: agent_number_version
+  mysql_number: agent_number_version
+  redis_number: agent_number_version
+  env_server: --processes 2 --stats :3031 --stats-http
+  env_beta: --concurrency=2
+  env_chain: --concurrency=12
+  env_es: -P gevent --concurrency=64
+  env_high: -P gevent --concurrency=121
+  env_other: --concurrency=10
+  env_sca: -P gevent --concurrency=10
+
+develop:
+  dev: true
+  redisCpu: "1"
+  redisMemory: 2000Mi
+  mysqlCpu: "1"
+  mysqlMemory: 2000Mi
+  mysqlLifecycle: "sleep 30; curl -o /tmp/hook_version.sql https://huoqi-public.oss-cn-beijing.aliyuncs.com/iast/hook_version.sql; cat /tmp/*.sql | mysql -uroot -p'dongtai-iast' --default-character-set=utf8mb4 dongtai_webapi"
+
+env:
+  METRICS: "true"
+  GEVENT: "TRUE"
+  CPROFILEF: "TRUE"
+  DOC: "TRUE"
+  SAVEEYE: "TRUE"
+  DJANGOSILK: "TRUE"
+  SILKY_PYTHON_PROFILER_BINARY: "True"
+  DJANGOSILKPATH: 9671ccbd0c655fda78354dda754c9c4fb7111b7c18751b25ea8930ab87c84f87
+  DB_ENGINE: mysql
+  DB_NAME: db.mysql
\ No newline at end of file
diff --git a/.github/workflows/deploy-dev.yaml b/.github/workflows/deploy-dev.yaml
index a48b7ce18..6944bcc77 100644
--- a/.github/workflows/deploy-dev.yaml
+++ b/.github/workflows/deploy-dev.yaml
@@ -11,9 +11,6 @@ on:
       agent_version:
         required: true
         type: string
-      # server_version:
-      #   required: true
-      #   type: string
 
 jobs:
   Deploy-to-dev:
@@ -27,12 +24,6 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      # - name: Checkout
-      #   if: ${{ inputs.server_version }}
-      #   uses: actions/checkout@v3
-      #   with:
-      #     ref: ${{ inputs.server_version }}
-
       - name: Get the release version
         id: version
         run: echo ::set-output name=GITHUB_REF::${GITHUB_REF##*/}
@@ -86,18 +77,18 @@ jobs:
           push: true
           platforms: linux/amd64
           tags: | 
-            registry.cn-hongkong.aliyuncs.com/secnium/dongtai-server-test:latest-${{ steps.version.outputs.GITHUB_REF }}
-            registry.cn-hongkong.aliyuncs.com/secnium/dongtai-server-test:1.0.${{github.run_number}}-${{ steps.version.outputs.GITHUB_REF }}
+            registry.cn-hongkong.aliyuncs.com/secnium/dongtai-server:${{ steps.version.outputs.GITHUB_REF }}-latest
+            registry.cn-hongkong.aliyuncs.com/secnium/dongtai-server:${{ steps.version.outputs.GITHUB_REF }}-1.0.${{github.run_number}}
 
-      - uses: actions/checkout@master
-      - name: deploy to cluster
-        uses: wahyd4/kubectl-helm-action@master
-        env:
-          KUBE_CONFIG_DATA: ${{ secrets.KUBE_CONFIG_TEST_DATA }}
-        with:
-          args: |
-            if [ ${{ steps.version.outputs.GITHUB_REF }} = beta ] ; then find .github/deploy/deploy-dongtai-server-test.yml -type f -exec sed -i 's/iast-test/iast-beta/g' {} \; 
-            elif [ ${{ steps.version.outputs.GITHUB_REF }} = develop ] ; then echo testing !!!
-            else find .github/deploy/deploy-dongtai-server-test.yml -type f -exec sed -i 's/iast-test/iast-main/g' {} \; ;fi
-            find .github/deploy/deploy-dongtai-server-test.yml -type f -exec sed -i 's/VERSION/1.0.${{github.run_number}}-${{ steps.version.outputs.GITHUB_REF }}/g' {} \;
-            ls .github/deploy/deploy-dongtai-server-test.yml | xargs -I {} kubectl apply -f {}
+#      - uses: actions/checkout@master
+#      - name: deploy to cluster
+#        uses: wahyd4/kubectl-helm-action@master
+#        env:
+#          KUBE_CONFIG_DATA: ${{ secrets.KUBE_CONFIG_TEST_DATA }}
+#        with:
+#          args: |
+#            if [ ${{ steps.version.outputs.GITHUB_REF }} = beta ] ; then find .github/deploy/deploy-dongtai-server-test.yml -type f -exec sed -i 's/iast-test/iast-beta/g' {} \;
+#            elif [ ${{ steps.version.outputs.GITHUB_REF }} = develop ] ; then echo testing !!!
+#            else find .github/deploy/deploy-dongtai-server-test.yml -type f -exec sed -i 's/iast-test/iast-main/g' {} \; ;fi
+#            find .github/deploy/deploy-dongtai-server-test.yml -type f -exec sed -i 's/VERSION/1.0.${{github.run_number}}-${{ steps.version.outputs.GITHUB_REF }}/g' {} \;
+#            ls .github/deploy/deploy-dongtai-server-test.yml | xargs -I {} kubectl apply -f {}
diff --git a/deploy/kubernetes/helm/templates/service/dongtai-logstash-svc.yaml b/deploy/kubernetes/helm/templates/service/dongtai-logstash-svc.yaml
index 106b66656..0533e03b9 100644
--- a/deploy/kubernetes/helm/templates/service/dongtai-logstash-svc.yaml
+++ b/deploy/kubernetes/helm/templates/service/dongtai-logstash-svc.yaml
@@ -8,13 +8,13 @@ metadata:
 spec:
   type: ClusterIP
   ports:
-    - name: agent-http
-      port: 8082
-      targetPort: 8082
-      protocol: TCP
-    - name: log-http
-      port: 8083
-      targetPort: 8083
-      protocol: TCP
+  - name: http-agent
+    port: 8082
+    protocol: TCP
+    targetPort: 8082
+  - name: tcp-log
+    port: 8083
+    protocol: TCP
+    targetPort: 8083
   selector:
     app: {{ template "dongtai.fullname" . }}-logstash
\ No newline at end of file
diff --git a/deploy/kubernetes/helm/templates/service/dongtai-mysql.yaml b/deploy/kubernetes/helm/templates/service/dongtai-mysql.yaml
index e510f2c31..843653249 100644
--- a/deploy/kubernetes/helm/templates/service/dongtai-mysql.yaml
+++ b/deploy/kubernetes/helm/templates/service/dongtai-mysql.yaml
@@ -11,7 +11,8 @@ spec:
   selector:
     app: {{ template "dongtai.fullname" . }}-mysql
   ports:
-      - protocol: TCP
-        port: 3306
-        targetPort: 3306
+  - name: tcp-mysql
+    port: 3306
+    protocol: TCP
+    targetPort: 3306
 {{- end -}}
\ No newline at end of file
diff --git a/deploy/kubernetes/helm/templates/service/dongtai-redis.yaml b/deploy/kubernetes/helm/templates/service/dongtai-redis.yaml
index df7a8f00f..fedb94067 100644
--- a/deploy/kubernetes/helm/templates/service/dongtai-redis.yaml
+++ b/deploy/kubernetes/helm/templates/service/dongtai-redis.yaml
@@ -9,9 +9,10 @@ metadata:
   namespace: {{.Release.Namespace}}
 spec:
   ports:
-    - port: 6379
-      protocol: TCP
-      targetPort: 6379
+  - name: redis-tcp
+    port: 6379
+    protocol: TCP
+    targetPort: 6379
   selector:
     app: {{ template "dongtai.fullname" . }}-redis
 {{- end -}}
\ No newline at end of file
diff --git a/deploy/kubernetes/helm/templates/service/dongtai-server-svc.yaml b/deploy/kubernetes/helm/templates/service/dongtai-server-svc.yaml
index 852ebaa92..2e4e837bc 100644
--- a/deploy/kubernetes/helm/templates/service/dongtai-server-svc.yaml
+++ b/deploy/kubernetes/helm/templates/service/dongtai-server-svc.yaml
@@ -6,7 +6,8 @@ metadata:
   namespace: {{.Release.Namespace}}
 spec:
   ports:
-    - port: 80
+    - name: http
+      port: 80
       protocol: TCP
       targetPort: 8000
   selector:
diff --git a/deploy/kubernetes/helm/templates/service/dongtai-web-svc.yaml b/deploy/kubernetes/helm/templates/service/dongtai-web-svc.yaml
index a3e101681..3b93ec39f 100644
--- a/deploy/kubernetes/helm/templates/service/dongtai-web-svc.yaml
+++ b/deploy/kubernetes/helm/templates/service/dongtai-web-svc.yaml
@@ -10,7 +10,8 @@ metadata:
   namespace: {{ .Release.Namespace }}
 spec:
   ports:
-    - port: 80
+    - name: http
+      port: 80
       protocol: TCP
       targetPort: 80
       {{- if (eq .Values.accessType "NodePort") }}

From 94eef91c476faf1559f5c67402759b3e015144be Mon Sep 17 00:00:00 2001
From: st1020 
Date: Wed, 12 Jul 2023 17:48:05 +0800
Subject: [PATCH 115/161] pref: improve strategy types performance

---
 dongtai_web/views/strategys_type.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/dongtai_web/views/strategys_type.py b/dongtai_web/views/strategys_type.py
index 78dc92155..eef7fe536 100644
--- a/dongtai_web/views/strategys_type.py
+++ b/dongtai_web/views/strategys_type.py
@@ -45,7 +45,9 @@ class StrategyType(UserEndPoint):
         response_schema=_ResponseSerializer,
     )
     def get(self, request):
-        queryset = IastStrategyModel.objects.filter(state="enable")
+        queryset = IastStrategyModel.objects.filter(
+            state="enable"
+        ).select_related("level")
         allType = IastVulLevel.objects.all().order_by("id")
         result = []
         curTyp = {}

From 8f135a7e0f3dd441de25d690590737e970d71a03 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Wed, 12 Jul 2023 17:58:50 +0800
Subject: [PATCH 116/161] pref: improve hook rules performance

---
 dongtai_web/views/engine_hook_rules.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dongtai_web/views/engine_hook_rules.py b/dongtai_web/views/engine_hook_rules.py
index 1f37c9e3f..bbb67a454 100644
--- a/dongtai_web/views/engine_hook_rules.py
+++ b/dongtai_web/views/engine_hook_rules.py
@@ -126,6 +126,7 @@ def get(self, request):
             rule_queryset = HookStrategy.objects.filter(q)
             page_summary, queryset = self.get_paginator(
                 rule_queryset.order_by('-id'), page=page, page_size=page_size)
+            queryset = queryset.select_related("hooktype")
             data = HookRuleSerializer(queryset, many=True).data
             return R.success(data=data, page=page_summary)
         except Exception as e:

From 284b31b85ace5005622d2e1fd9b9b76f926feb6e Mon Sep 17 00:00:00 2001
From: tscuite 
Date: Wed, 12 Jul 2023 18:28:55 +0800
Subject: [PATCH 117/161] feat: update ci

---
 .github/deploy/devops.yml         | 54 -------------------------------
 .github/workflows/deploy-dev.yaml | 28 +++++++++-------
 2 files changed, 16 insertions(+), 66 deletions(-)
 delete mode 100644 .github/deploy/devops.yml

diff --git a/.github/deploy/devops.yml b/.github/deploy/devops.yml
deleted file mode 100644
index d8eb5d3f6..000000000
--- a/.github/deploy/devops.yml
+++ /dev/null
@@ -1,54 +0,0 @@
-images: registry.cn-beijing.aliyuncs.com/huoxian_pub    # or   dongtai
-tag:
-imagePullPolicy: Always
-replicaCount: 1
-cpu: 500m
-memory: 1000Mi
-logging_level: DEBUG # DEBUG, INFO
-healthcheck: true
-
-mysql:
-  host:
-
-sca:
-  sca_token:
-
-storage:
-  storageClassName: alicloud-nas-subpath
-
-csrfTrustOrigins:
-
-build:
-  agent_number: agent_number_version
-  web_number: agent_number_version
-  server_number: agent_number_version
-  logstash_number: agent_number_version
-  mysql_number: agent_number_version
-  redis_number: agent_number_version
-  env_server: --processes 2 --stats :3031 --stats-http
-  env_beta: --concurrency=2
-  env_chain: --concurrency=12
-  env_es: -P gevent --concurrency=64
-  env_high: -P gevent --concurrency=121
-  env_other: --concurrency=10
-  env_sca: -P gevent --concurrency=10
-
-develop:
-  dev: true
-  redisCpu: "1"
-  redisMemory: 2000Mi
-  mysqlCpu: "1"
-  mysqlMemory: 2000Mi
-  mysqlLifecycle: "sleep 30; curl -o /tmp/hook_version.sql https://huoqi-public.oss-cn-beijing.aliyuncs.com/iast/hook_version.sql; cat /tmp/*.sql | mysql -uroot -p'dongtai-iast' --default-character-set=utf8mb4 dongtai_webapi"
-
-env:
-  METRICS: "true"
-  GEVENT: "TRUE"
-  CPROFILEF: "TRUE"
-  DOC: "TRUE"
-  SAVEEYE: "TRUE"
-  DJANGOSILK: "TRUE"
-  SILKY_PYTHON_PROFILER_BINARY: "True"
-  DJANGOSILKPATH: 9671ccbd0c655fda78354dda754c9c4fb7111b7c18751b25ea8930ab87c84f87
-  DB_ENGINE: mysql
-  DB_NAME: db.mysql
\ No newline at end of file
diff --git a/.github/workflows/deploy-dev.yaml b/.github/workflows/deploy-dev.yaml
index 6944bcc77..74185c3e2 100644
--- a/.github/workflows/deploy-dev.yaml
+++ b/.github/workflows/deploy-dev.yaml
@@ -80,15 +80,19 @@ jobs:
             registry.cn-hongkong.aliyuncs.com/secnium/dongtai-server:${{ steps.version.outputs.GITHUB_REF }}-latest
             registry.cn-hongkong.aliyuncs.com/secnium/dongtai-server:${{ steps.version.outputs.GITHUB_REF }}-1.0.${{github.run_number}}
 
-#      - uses: actions/checkout@master
-#      - name: deploy to cluster
-#        uses: wahyd4/kubectl-helm-action@master
-#        env:
-#          KUBE_CONFIG_DATA: ${{ secrets.KUBE_CONFIG_TEST_DATA }}
-#        with:
-#          args: |
-#            if [ ${{ steps.version.outputs.GITHUB_REF }} = beta ] ; then find .github/deploy/deploy-dongtai-server-test.yml -type f -exec sed -i 's/iast-test/iast-beta/g' {} \;
-#            elif [ ${{ steps.version.outputs.GITHUB_REF }} = develop ] ; then echo testing !!!
-#            else find .github/deploy/deploy-dongtai-server-test.yml -type f -exec sed -i 's/iast-test/iast-main/g' {} \; ;fi
-#            find .github/deploy/deploy-dongtai-server-test.yml -type f -exec sed -i 's/VERSION/1.0.${{github.run_number}}-${{ steps.version.outputs.GITHUB_REF }}/g' {} \;
-#            ls .github/deploy/deploy-dongtai-server-test.yml | xargs -I {} kubectl apply -f {}
+      - uses: actions/checkout@master
+      - name: deploy to cluster
+        uses: wahyd4/kubectl-helm-action@master
+        env:
+          KUBE_CONFIG_DATA: ${{ secrets.KUBE_CONFIG_TEST_DATA }}
+        with:
+          args: |
+            git clone https://github.com/HXSecurity/DongTai.git
+            
+            if [ ${{ steps.version.outputs.GITHUB_REF }} = develop ];
+            then helm upgrade --install huoxian --create-namespace -n iast-test ./deploy/kubernetes/helm/ --set mysql.host=iast-mysql-test.huoxian.cn \
+            --set tag=${{ steps.version.outputs.GITHUB_REF }}-latest --set build.server_number=${{github.run_number}} --values https://charts.dongtai.io/devops.yaml
+            else
+            helm upgrade --install huoxian --create-namespace -n iast-${{ steps.version.outputs.GITHUB_REF }} ./deploy/kubernetes/helm/ --set mysql.host=iast-mysql-test.huoxian.cn \
+            --set tag=${{ steps.version.outputs.GITHUB_REF }}-latest --set build.server_number=${{github.run_number}} --values https://charts.dongtai.io/devops.yaml
+            fi
\ No newline at end of file

From 564c4606399345ef7b85fb7c1dbd5d0facaea286 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Wed, 12 Jul 2023 18:31:49 +0800
Subject: [PATCH 118/161] pref: improve hook rules performance

---
 dongtai_web/views/engine_hook_rules.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/dongtai_web/views/engine_hook_rules.py b/dongtai_web/views/engine_hook_rules.py
index bbb67a454..ed02306dc 100644
--- a/dongtai_web/views/engine_hook_rules.py
+++ b/dongtai_web/views/engine_hook_rules.py
@@ -126,7 +126,10 @@ def get(self, request):
             rule_queryset = HookStrategy.objects.filter(q)
             page_summary, queryset = self.get_paginator(
                 rule_queryset.order_by('-id'), page=page, page_size=page_size)
-            queryset = queryset.select_related("hooktype")
+            if rule_type == 4:
+                queryset = queryset.select_related("strategy")
+            else:
+                queryset = queryset.select_related("hooktype")
             data = HookRuleSerializer(queryset, many=True).data
             return R.success(data=data, page=page_summary)
         except Exception as e:

From eb4983c0375fa6988ac255686fb7b188dcfcee21 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Wed, 12 Jul 2023 18:37:14 +0800
Subject: [PATCH 119/161] fix: update project status error

---
 dongtai_common/models/project.py         | 2 +-
 dongtai_engine/plugins/project_status.py | 9 ++-------
 2 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/dongtai_common/models/project.py b/dongtai_common/models/project.py
index 79779960e..942f7a1a1 100644
--- a/dongtai_common/models/project.py
+++ b/dongtai_common/models/project.py
@@ -78,7 +78,7 @@ class IastProject(models.Model):
     template = models.ForeignKey(IastProjectTemplate, models.DO_NOTHING)
     enable_log = models.BooleanField(null=True)
     log_level = models.CharField(max_length=511, null=True, blank=True)
-    last_has_online_agent_time = models.IntegerField(default=-1)
+    last_has_online_agent_time = models.IntegerField(default=lambda: int(time.time()))
     status = models.IntegerField(default=0, choices=ProjectStatus.choices)
 
     class Meta:
diff --git a/dongtai_engine/plugins/project_status.py b/dongtai_engine/plugins/project_status.py
index ea58e06c7..80471cad8 100644
--- a/dongtai_engine/plugins/project_status.py
+++ b/dongtai_engine/plugins/project_status.py
@@ -39,15 +39,10 @@ def update_project_status() -> None:
         old_status = project.status
 
         if online_agent_count == 0:
-            if project.last_has_online_agent_time == -1:
-                project.last_has_online_agent_time = int(time())
-                project.save(update_fields=("last_has_online_agent_time"))
-                continue
-
             dt = int(time()) - project.last_has_online_agent_time
             project_warning_time = get_project_warning_time()
-            error_time = project_warning_time["error_time"]
-            offline_time = project_warning_time["offline_time"]
+            error_time = project_warning_time["error_time"] * 60 * 60 * 24
+            offline_time = project_warning_time["offline_time"] * 60 * 60 * 24
 
             if dt < error_time:
                 project.status = ProjectStatus.NORMAL

From 09e66004081605e0a4416ac47e15f91825c769cb Mon Sep 17 00:00:00 2001
From: tscuite 
Date: Wed, 12 Jul 2023 18:48:28 +0800
Subject: [PATCH 120/161] feat: update ci

---
 .github/workflows/deploy-dev.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/deploy-dev.yaml b/.github/workflows/deploy-dev.yaml
index 74185c3e2..1492332d6 100644
--- a/.github/workflows/deploy-dev.yaml
+++ b/.github/workflows/deploy-dev.yaml
@@ -94,5 +94,5 @@ jobs:
             --set tag=${{ steps.version.outputs.GITHUB_REF }}-latest --set build.server_number=${{github.run_number}} --values https://charts.dongtai.io/devops.yaml
             else
             helm upgrade --install huoxian --create-namespace -n iast-${{ steps.version.outputs.GITHUB_REF }} ./deploy/kubernetes/helm/ --set mysql.host=iast-mysql-test.huoxian.cn \
-            --set tag=${{ steps.version.outputs.GITHUB_REF }}-latest --set build.server_number=${{github.run_number}} --values https://charts.dongtai.io/devops.yaml
+            --set tag=${{ steps.version.outputs.GITHUB_REF }}-latest --set build.server_number=iast${{github.run_number}} --values https://charts.dongtai.io/devops.yaml
             fi
\ No newline at end of file

From 5137ec65b2e17c87763a4e9ce28448278c5ee24c Mon Sep 17 00:00:00 2001
From: tscuite 
Date: Wed, 12 Jul 2023 18:53:12 +0800
Subject: [PATCH 121/161] feat: update ci

---
 .github/workflows/deploy-dev.yaml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/deploy-dev.yaml b/.github/workflows/deploy-dev.yaml
index 1492332d6..57bcf1157 100644
--- a/.github/workflows/deploy-dev.yaml
+++ b/.github/workflows/deploy-dev.yaml
@@ -88,11 +88,10 @@ jobs:
         with:
           args: |
             git clone https://github.com/HXSecurity/DongTai.git
-            
             if [ ${{ steps.version.outputs.GITHUB_REF }} = develop ];
-            then helm upgrade --install huoxian --create-namespace -n iast-test ./deploy/kubernetes/helm/ --set mysql.host=iast-mysql-test.huoxian.cn \
-            --set tag=${{ steps.version.outputs.GITHUB_REF }}-latest --set build.server_number=${{github.run_number}} --values https://charts.dongtai.io/devops.yaml
+            then helm upgrade --install huoxian --create-namespace -n iast-test ./DongTai/deploy/kubernetes/helm/ --set mysql.host=iast-mysql-test.huoxian.cn \
+            --set tag=${{ steps.version.outputs.GITHUB_REF }}-latest --set build.server_number=iast${{github.run_number}} --values https://charts.dongtai.io/devops.yaml
             else
-            helm upgrade --install huoxian --create-namespace -n iast-${{ steps.version.outputs.GITHUB_REF }} ./deploy/kubernetes/helm/ --set mysql.host=iast-mysql-test.huoxian.cn \
+            helm upgrade --install huoxian --create-namespace -n iast-${{ steps.version.outputs.GITHUB_REF }} ./DongTai/deploy/kubernetes/helm/ --set mysql.host=iast-mysql-test.huoxian.cn \
             --set tag=${{ steps.version.outputs.GITHUB_REF }}-latest --set build.server_number=iast${{github.run_number}} --values https://charts.dongtai.io/devops.yaml
             fi
\ No newline at end of file

From 8a6743469b5cd198e99b27bd4305f596e22aabce Mon Sep 17 00:00:00 2001
From: tscuite 
Date: Wed, 12 Jul 2023 20:58:51 +0800
Subject: [PATCH 122/161] feat: update ci

---
 .../helm/templates/deployments/dongtai-worker-sca.yaml         | 2 +-
 .../helm/templates/deployments/dongtai-worker-task.yaml        | 3 ---
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-sca.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-sca.yaml
index 07f5b9a21..d61d56e11 100644
--- a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-sca.yaml
+++ b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-sca.yaml
@@ -35,7 +35,7 @@ spec:
           args: [ "worker-sca" ]
           env:
             - name: DONGTAI_CONCURRENCY
-              value: {{.Values.build.env_server}}
+              value: {{.Values.build.env_sca}}
           {{- include "deploy.config" . | nindent 10 }}
           {{- if .Values.healthcheck }}
           {{- include "deploy.Probe" . | nindent 10 }}
diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-task.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-task.yaml
index e43238e84..3369e7703 100644
--- a/deploy/kubernetes/helm/templates/deployments/dongtai-worker-task.yaml
+++ b/deploy/kubernetes/helm/templates/deployments/dongtai-worker-task.yaml
@@ -34,7 +34,4 @@ spec:
           command: [ "/bin/bash","/opt/dongtai/deploy/docker/entrypoint.sh" ]
           args: [ "beat" ]
           {{- include "deploy.config" . | nindent 10 }}
-          # {{- if .Values.healthcheck }}
-          # {{- include "deploy.Probe" . | nindent 10 }}
-          # {{- end }}
       {{- include "deploy.config.vo" . | nindent 6 }}
\ No newline at end of file

From 99f21a979ac72ba705320bddc59f66ac4bda6db6 Mon Sep 17 00:00:00 2001
From: tscuite 
Date: Wed, 12 Jul 2023 21:39:22 +0800
Subject: [PATCH 123/161] feat: update ci

---
 .github/workflows/deploy-dev.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/deploy-dev.yaml b/.github/workflows/deploy-dev.yaml
index 57bcf1157..d979f2b2e 100644
--- a/.github/workflows/deploy-dev.yaml
+++ b/.github/workflows/deploy-dev.yaml
@@ -89,9 +89,11 @@ jobs:
           args: |
             git clone https://github.com/HXSecurity/DongTai.git
             if [ ${{ steps.version.outputs.GITHUB_REF }} = develop ];
-            then helm upgrade --install huoxian --create-namespace -n iast-test ./DongTai/deploy/kubernetes/helm/ --set mysql.host=iast-mysql-test.huoxian.cn \
+            then helm upgrade --install huoxian --create-namespace -n iast-test ./DongTai/deploy/kubernetes/helm/ \
+            --set sca.sca_token=${{ secrets.TOKEN_SCA }} --set usb.usb_token=${{ secrets.TOKEN_SCA }} --set mysql.host=iast-mysql-test.huoxian.cn \
             --set tag=${{ steps.version.outputs.GITHUB_REF }}-latest --set build.server_number=iast${{github.run_number}} --values https://charts.dongtai.io/devops.yaml
             else
-            helm upgrade --install huoxian --create-namespace -n iast-${{ steps.version.outputs.GITHUB_REF }} ./DongTai/deploy/kubernetes/helm/ --set mysql.host=iast-mysql-test.huoxian.cn \
+            helm upgrade --install huoxian --create-namespace -n iast-${{ steps.version.outputs.GITHUB_REF }} ./DongTai/deploy/kubernetes/helm/ \
+            --set sca.sca_token=${{ secrets.TOKEN_SCA }} --set usb.usb_token=${{ secrets.TOKEN_SCA }} --set mysql.host=iast-mysql-test.huoxian.cn \
             --set tag=${{ steps.version.outputs.GITHUB_REF }}-latest --set build.server_number=iast${{github.run_number}} --values https://charts.dongtai.io/devops.yaml
             fi
\ No newline at end of file

From b507aaa9800d7c6f6f940112299fbe5225b8e9b7 Mon Sep 17 00:00:00 2001
From: tscuite 
Date: Thu, 13 Jul 2023 10:05:57 +0800
Subject: [PATCH 124/161] feat: update ci

---
 .../kubernetes/helm/templates/deployments/dongtai-server.yaml   | 2 +-
 deploy/kubernetes/helm/values.yaml                              | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-server.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-server.yaml
index 420bbbc20..96b2c18e7 100644
--- a/deploy/kubernetes/helm/templates/deployments/dongtai-server.yaml
+++ b/deploy/kubernetes/helm/templates/deployments/dongtai-server.yaml
@@ -37,7 +37,7 @@ spec:
           env:
             - name: DONGTAI_CONCURRENCY
               value: {{.Values.build.env_server}}
-            {{- range $key, $value :=  .Values.serverEnv }}
+            {{- range $key, $value :=  .Values.env }}
             - name: {{ $key }}
               value: {{ $value | quote }}
             {{- end }}
diff --git a/deploy/kubernetes/helm/values.yaml b/deploy/kubernetes/helm/values.yaml
index aa3347823..160a9aa85 100644
--- a/deploy/kubernetes/helm/values.yaml
+++ b/deploy/kubernetes/helm/values.yaml
@@ -22,6 +22,8 @@ nodeSelector:
   kubernetes.io/arch: amd64
 podLabels: {}
   # example: "test"
+env: {}
+  # example: "test"
 
 skipMysql: false
 mysql:

From 92c78336fb3f54019965e2230e40908218297158 Mon Sep 17 00:00:00 2001
From: tscuite 
Date: Thu, 13 Jul 2023 10:19:58 +0800
Subject: [PATCH 125/161] feat: update ci

---
 .github/workflows/deploy-dev.yaml | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/deploy-dev.yaml b/.github/workflows/deploy-dev.yaml
index d979f2b2e..0ba7e24a6 100644
--- a/.github/workflows/deploy-dev.yaml
+++ b/.github/workflows/deploy-dev.yaml
@@ -54,7 +54,7 @@ jobs:
           access-key-secret: ${{ secrets.CHART_OSS_ACCESS_KEY_SECRET }}
           
       - name: Download Agent dev
-        if: ${{ steps.version.outputs.GITHUB_REF }} == develop || ${{ steps.version.outputs.GITHUB_REF }} == beta || ${{ steps.version.outputs.GITHUB_REF }} == main
+#        if: ${{ steps.version.outputs.GITHUB_REF }} == develop || ${{ steps.version.outputs.GITHUB_REF }} == beta || ${{ steps.version.outputs.GITHUB_REF }} == main
         run: |
           if [ ${{ steps.version.outputs.GITHUB_REF }} = develop ] ; then ossutil cp oss://dongtai-helm-charts/agent_test/java/latest/ ./ --include "*.jar" -r
           else  ossutil cp oss://dongtai-helm-charts/agent_${{ steps.version.outputs.GITHUB_REF }}/java/latest/ ./ --include "*.jar" -r
@@ -87,13 +87,11 @@ jobs:
           KUBE_CONFIG_DATA: ${{ secrets.KUBE_CONFIG_TEST_DATA }}
         with:
           args: |
+            ns=main
             git clone https://github.com/HXSecurity/DongTai.git
-            if [ ${{ steps.version.outputs.GITHUB_REF }} = develop ];
-            then helm upgrade --install huoxian --create-namespace -n iast-test ./DongTai/deploy/kubernetes/helm/ \
+            if [ ${{ steps.version.outputs.GITHUB_REF }} = develop ] ; then ns= test
+            elif [ ${{ steps.version.outputs.GITHUB_REF }} = beta ] ; then ns= beta
+            else echo "start main" ;fi
+            helm upgrade --install huoxian --create-namespace -n iast-${ns} ./DongTai/deploy/kubernetes/helm/ \
             --set sca.sca_token=${{ secrets.TOKEN_SCA }} --set usb.usb_token=${{ secrets.TOKEN_SCA }} --set mysql.host=iast-mysql-test.huoxian.cn \
-            --set tag=${{ steps.version.outputs.GITHUB_REF }}-latest --set build.server_number=iast${{github.run_number}} --values https://charts.dongtai.io/devops.yaml
-            else
-            helm upgrade --install huoxian --create-namespace -n iast-${{ steps.version.outputs.GITHUB_REF }} ./DongTai/deploy/kubernetes/helm/ \
-            --set sca.sca_token=${{ secrets.TOKEN_SCA }} --set usb.usb_token=${{ secrets.TOKEN_SCA }} --set mysql.host=iast-mysql-test.huoxian.cn \
-            --set tag=${{ steps.version.outputs.GITHUB_REF }}-latest --set build.server_number=iast${{github.run_number}} --values https://charts.dongtai.io/devops.yaml
-            fi
\ No newline at end of file
+            --set tag=${{ steps.version.outputs.GITHUB_REF }}-latest --set build.server_number=iast${{github.run_number}} --values https://charts.dongtai.io/devops.yaml
\ No newline at end of file

From fedd614c88dbeef0486c0a5260e7688a26848b13 Mon Sep 17 00:00:00 2001
From: tscuite 
Date: Thu, 13 Jul 2023 10:27:58 +0800
Subject: [PATCH 126/161] feat: update ci

---
 .github/workflows/deploy-dev.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/deploy-dev.yaml b/.github/workflows/deploy-dev.yaml
index 0ba7e24a6..f2a8b742a 100644
--- a/.github/workflows/deploy-dev.yaml
+++ b/.github/workflows/deploy-dev.yaml
@@ -87,11 +87,11 @@ jobs:
           KUBE_CONFIG_DATA: ${{ secrets.KUBE_CONFIG_TEST_DATA }}
         with:
           args: |
-            ns=main
+            echo "helm_ns=main" >> $GITHUB_ENV
             git clone https://github.com/HXSecurity/DongTai.git
-            if [ ${{ steps.version.outputs.GITHUB_REF }} = develop ] ; then ns= test
-            elif [ ${{ steps.version.outputs.GITHUB_REF }} = beta ] ; then ns= beta
+            if [ ${{ steps.version.outputs.GITHUB_REF }} = develop ] ; then echo "helm_ns=test" >> $GITHUB_ENV
+            elif [ ${{ steps.version.outputs.GITHUB_REF }} = beta ] ; then echo "helm_ns=beta" >> $GITHUB_ENV
             else echo "start main" ;fi
-            helm upgrade --install huoxian --create-namespace -n iast-${ns} ./DongTai/deploy/kubernetes/helm/ \
+            helm upgrade --install huoxian --create-namespace -n iast-${{ env.helm_ns }} ./DongTai/deploy/kubernetes/helm/ \
             --set sca.sca_token=${{ secrets.TOKEN_SCA }} --set usb.usb_token=${{ secrets.TOKEN_SCA }} --set mysql.host=iast-mysql-test.huoxian.cn \
             --set tag=${{ steps.version.outputs.GITHUB_REF }}-latest --set build.server_number=iast${{github.run_number}} --values https://charts.dongtai.io/devops.yaml
\ No newline at end of file

From d5f4c6892c2b8091464596bbe304326108c095bd Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Thu, 13 Jul 2023 10:35:58 +0800
Subject: [PATCH 127/161] fix: agent download template id.

---
 dongtai_protocol/views/agent_download.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dongtai_protocol/views/agent_download.py b/dongtai_protocol/views/agent_download.py
index f10bf3bbe..f1c020f50 100644
--- a/dongtai_protocol/views/agent_download.py
+++ b/dongtai_protocol/views/agent_download.py
@@ -327,7 +327,7 @@ def get(self, request):
                                                        'V1.0')
             language = request.query_params.get('language')
             department_token = request.query_params.get('department_token')
-            template_id = request.query_params.get('template_id')
+            template_id = request.query_params.get('template_id', 5)
             user_token = request.query_params.get('token', None)
             if department_token:
                 final_token = department_token

From 6e181e377364ec0337f65ce184a27c8384abc360 Mon Sep 17 00:00:00 2001
From: tscuite 
Date: Thu, 13 Jul 2023 10:42:53 +0800
Subject: [PATCH 128/161] feat: update ci

---
 .github/workflows/deploy-dev.yaml | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/deploy-dev.yaml b/.github/workflows/deploy-dev.yaml
index f2a8b742a..395390826 100644
--- a/.github/workflows/deploy-dev.yaml
+++ b/.github/workflows/deploy-dev.yaml
@@ -80,18 +80,20 @@ jobs:
             registry.cn-hongkong.aliyuncs.com/secnium/dongtai-server:${{ steps.version.outputs.GITHUB_REF }}-latest
             registry.cn-hongkong.aliyuncs.com/secnium/dongtai-server:${{ steps.version.outputs.GITHUB_REF }}-1.0.${{github.run_number}}
 
-      - uses: actions/checkout@master
+      - name: Set the value
+        id: release
+        run: |
+          if [ ${{ steps.version.outputs.GITHUB_REF }} = develop ] ; then echo "helm_ns=test" >> $GITHUB_ENV
+          elif [ ${{ steps.version.outputs.GITHUB_REF }} = beta ] ; then echo "helm_ns=beta" >> $GITHUB_ENV
+          else echo "helm_ns=main" >> $GITHUB_ENV ;fi
+
       - name: deploy to cluster
         uses: wahyd4/kubectl-helm-action@master
         env:
           KUBE_CONFIG_DATA: ${{ secrets.KUBE_CONFIG_TEST_DATA }}
         with:
           args: |
-            echo "helm_ns=main" >> $GITHUB_ENV
             git clone https://github.com/HXSecurity/DongTai.git
-            if [ ${{ steps.version.outputs.GITHUB_REF }} = develop ] ; then echo "helm_ns=test" >> $GITHUB_ENV
-            elif [ ${{ steps.version.outputs.GITHUB_REF }} = beta ] ; then echo "helm_ns=beta" >> $GITHUB_ENV
-            else echo "start main" ;fi
             helm upgrade --install huoxian --create-namespace -n iast-${{ env.helm_ns }} ./DongTai/deploy/kubernetes/helm/ \
             --set sca.sca_token=${{ secrets.TOKEN_SCA }} --set usb.usb_token=${{ secrets.TOKEN_SCA }} --set mysql.host=iast-mysql-test.huoxian.cn \
             --set tag=${{ steps.version.outputs.GITHUB_REF }}-latest --set build.server_number=iast${{github.run_number}} --values https://charts.dongtai.io/devops.yaml
\ No newline at end of file

From fa55646fbf7c9461a415bb64ee7d533ea2fa6649 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Thu, 13 Jul 2023 11:14:32 +0800
Subject: [PATCH 129/161] feat: disable drf spectacular log

---
 dongtai_conf/settings.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/dongtai_conf/settings.py b/dongtai_conf/settings.py
index eed0777ea..56e4aff27 100644
--- a/dongtai_conf/settings.py
+++ b/dongtai_conf/settings.py
@@ -529,6 +529,8 @@ def safe_execute(default, exception, function, *args):
       ),
     'COMPONENT_SPLIT_REQUEST':
     True,
+    'DISABLE_ERRORS_AND_WARNINGS':
+    True,
 }
 REST_FRAMEWORK['DEFAULT_SCHEMA_CLASS'] = 'drf_spectacular.openapi.AutoSchema'
 

From d8d6db40b5f9b37a385241646336da5995549a11 Mon Sep 17 00:00:00 2001
From: tscuite 
Date: Thu, 13 Jul 2023 11:22:16 +0800
Subject: [PATCH 130/161] feat: update ci

---
 .../helm/templates/deployments/dongtai-logtsash.yaml          | 2 +-
 deploy/kubernetes/helm/values.yaml                            | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-logtsash.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-logtsash.yaml
index 320f49c15..71b156f92 100644
--- a/deploy/kubernetes/helm/templates/deployments/dongtai-logtsash.yaml
+++ b/deploy/kubernetes/helm/templates/deployments/dongtai-logtsash.yaml
@@ -20,7 +20,7 @@ spec:
         sidecar.istio.io/inject: "true"
         sidecar.istio.io/interceptionMode: TPROXY
         {{- end }}
-        build_number: "{{ template "dongtai.fullname" . }}"
+        logstash_number: {{.Values.build.logstash_number}}
         {{- if not .Values.skipistio }}{{ include "dongtai.istiolabels" . | nindent 8}}{{ end }}
       labels:
         app: {{ template "dongtai.fullname" . }}-logstash
diff --git a/deploy/kubernetes/helm/values.yaml b/deploy/kubernetes/helm/values.yaml
index 160a9aa85..59946e29c 100644
--- a/deploy/kubernetes/helm/values.yaml
+++ b/deploy/kubernetes/helm/values.yaml
@@ -72,10 +72,10 @@ build:
   redis_number: agent_number_version
   env_server: --processes 4
   env_beta: --concurrency=2
-  env_chain: --concurrency=12
+  env_chain: -P gevent --concurrency=121
   env_es: -P gevent --concurrency=64
   env_high: -P gevent --concurrency=121
-  env_other: --concurrency=10
+  env_other: -P gevent --concurrency=10
   env_sca: -P gevent --concurrency=10
 
 develop:

From 62c78e0691a782181757e4358428a6f28fe51fb9 Mon Sep 17 00:00:00 2001
From: tscuite 
Date: Thu, 13 Jul 2023 11:39:10 +0800
Subject: [PATCH 131/161] feat: update ci timeout

---
 deploy/kubernetes/helm/templates/_helpers.tpl  | 12 ++++++------
 .../deployments/dongtai-logtsash.yaml          | 18 ++++++++++--------
 .../templates/deployments/dongtai-server.yaml  |  6 +++---
 3 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/deploy/kubernetes/helm/templates/_helpers.tpl b/deploy/kubernetes/helm/templates/_helpers.tpl
index d4b86bf85..a23b59108 100644
--- a/deploy/kubernetes/helm/templates/_helpers.tpl
+++ b/deploy/kubernetes/helm/templates/_helpers.tpl
@@ -152,9 +152,9 @@ readinessProbe:
     - celery -A dongtai_conf inspect ping -d celery@$(hostname)
   failureThreshold: 3
   initialDelaySeconds: 30
-  periodSeconds: 5
+  periodSeconds: 10
   successThreshold: 1
-  timeoutSeconds: 3
+  timeoutSeconds: 5
 livenessProbe:
   exec:
     command:
@@ -163,9 +163,9 @@ livenessProbe:
     - celery -A dongtai_conf inspect ping -d celery@$(hostname)
   failureThreshold: 3
   initialDelaySeconds: 30
-  periodSeconds: 5
+  periodSeconds: 10
   successThreshold: 1
-  timeoutSeconds: 3
+  timeoutSeconds: 5
 startupProbe:
   exec:
     command:
@@ -174,9 +174,9 @@ startupProbe:
     - celery -A dongtai_conf inspect ping -d celery@$(hostname)
   failureThreshold: 3
   initialDelaySeconds: 30
-  periodSeconds: 5
+  periodSeconds: 10
   successThreshold: 1
-  timeoutSeconds: 3
+  timeoutSeconds: 5
 {{- end -}}
 
 {{- define "deploy.resources" -}}
diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-logtsash.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-logtsash.yaml
index 71b156f92..15cdb08d7 100644
--- a/deploy/kubernetes/helm/templates/deployments/dongtai-logtsash.yaml
+++ b/deploy/kubernetes/helm/templates/deployments/dongtai-logtsash.yaml
@@ -61,27 +61,29 @@ spec:
           name: log-http
         {{- if .Values.healthcheck }}
         livenessProbe:
-          failureThreshold: 1
-          periodSeconds: 5
+          failureThreshold: 3
+          initialDelaySeconds: 30
+          periodSeconds: 10
           successThreshold: 1
           tcpSocket:
             port: 9600
-          timeoutSeconds: 1
+          timeoutSeconds: 5
         readinessProbe:
           failureThreshold: 3
           initialDelaySeconds: 30
-          periodSeconds: 5
+          periodSeconds: 10
           successThreshold: 1
           tcpSocket:
             port: 9600
-          timeoutSeconds: 1
+          timeoutSeconds: 5
         startupProbe:
-          failureThreshold: 40
-          periodSeconds: 5
+          failureThreshold: 3
+          initialDelaySeconds: 30
+          periodSeconds: 10
           successThreshold: 1
           tcpSocket:
             port: 9600
-          timeoutSeconds: 1
+          timeoutSeconds: 5
         {{- end }}
         resources:
           requests:
diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-server.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-server.yaml
index 96b2c18e7..58f2f6446 100644
--- a/deploy/kubernetes/helm/templates/deployments/dongtai-server.yaml
+++ b/deploy/kubernetes/helm/templates/deployments/dongtai-server.yaml
@@ -52,7 +52,7 @@ spec:
             initialDelaySeconds: 30
             periodSeconds: 10
             successThreshold: 1
-            timeoutSeconds: 3
+            timeoutSeconds: 5
           name: dongtai-server-container
           readinessProbe:
             failureThreshold: 3
@@ -63,7 +63,7 @@ spec:
             initialDelaySeconds: 30
             periodSeconds: 10
             successThreshold: 1
-            timeoutSeconds: 3
+            timeoutSeconds: 5
           startupProbe:
             failureThreshold: 3
             httpGet:
@@ -73,7 +73,7 @@ spec:
             initialDelaySeconds: 30
             periodSeconds: 10
             successThreshold: 1
-            timeoutSeconds: 3
+            timeoutSeconds: 5
           {{- end }}
       {{- include "deploy.config.vo" . | nindent 6 }}
       {{- if .Values.somaxconn }}

From 205c0f5c69e99c4a28ac90a17f36b841670d7693 Mon Sep 17 00:00:00 2001
From: tscuite 
Date: Thu, 13 Jul 2023 11:49:36 +0800
Subject: [PATCH 132/161] feat: update healthcheck

---
 deploy/kubernetes/helm/templates/_helpers.tpl | 26 +++++++------------
 .../helm/templates/data/dongtai-mysql.yaml    | 17 +++---------
 .../deployments/dongtai-logtsash.yaml         | 18 +++----------
 .../templates/deployments/dongtai-server.yaml | 18 +++----------
 deploy/kubernetes/helm/values.yaml            |  2 +-
 5 files changed, 20 insertions(+), 61 deletions(-)

diff --git a/deploy/kubernetes/helm/templates/_helpers.tpl b/deploy/kubernetes/helm/templates/_helpers.tpl
index a23b59108..8cf0b1c4a 100644
--- a/deploy/kubernetes/helm/templates/_helpers.tpl
+++ b/deploy/kubernetes/helm/templates/_helpers.tpl
@@ -142,7 +142,13 @@ initContainers:
     securityContext:
       privileged: true
 {{- end -}}
-
+{{- define "deploy.Probehealthcheck" -}}
+failureThreshold: 3
+initialDelaySeconds: 30
+periodSeconds: 10
+successThreshold: 1
+timeoutSeconds: 5
+{{- end -}}
 {{- define "deploy.Probe" -}}
 readinessProbe:
   exec:
@@ -150,33 +156,21 @@ readinessProbe:
     - bash
     - -c
     - celery -A dongtai_conf inspect ping -d celery@$(hostname)
-  failureThreshold: 3
-  initialDelaySeconds: 30
-  periodSeconds: 10
-  successThreshold: 1
-  timeoutSeconds: 5
+  {{- include "deploy.Probehealthcheck" . | nindent 2 }}
 livenessProbe:
   exec:
     command:
     - bash
     - -c
     - celery -A dongtai_conf inspect ping -d celery@$(hostname)
-  failureThreshold: 3
-  initialDelaySeconds: 30
-  periodSeconds: 10
-  successThreshold: 1
-  timeoutSeconds: 5
+  {{- include "deploy.Probehealthcheck" . | nindent 2 }}
 startupProbe:
   exec:
     command:
     - bash
     - -c
     - celery -A dongtai_conf inspect ping -d celery@$(hostname)
-  failureThreshold: 3
-  initialDelaySeconds: 30
-  periodSeconds: 10
-  successThreshold: 1
-  timeoutSeconds: 5
+  {{- include "deploy.Probehealthcheck" . | nindent 2 }}
 {{- end -}}
 
 {{- define "deploy.resources" -}}
diff --git a/deploy/kubernetes/helm/templates/data/dongtai-mysql.yaml b/deploy/kubernetes/helm/templates/data/dongtai-mysql.yaml
index e014e561a..7204c6d24 100644
--- a/deploy/kubernetes/helm/templates/data/dongtai-mysql.yaml
+++ b/deploy/kubernetes/helm/templates/data/dongtai-mysql.yaml
@@ -49,28 +49,17 @@ spec:
             name: tcp-mysql
         {{- if .Values.healthcheck }}
         livenessProbe:
-          failureThreshold: 3
-          initialDelaySeconds: 30
-          periodSeconds: 5
-          successThreshold: 1
+          {{- include "deploy.Probehealthcheck" . | nindent 10 }}
           tcpSocket:
             port: 3306
-          timeoutSeconds: 1
         readinessProbe:
-          failureThreshold: 3
-          initialDelaySeconds: 30
-          periodSeconds: 5
-          successThreshold: 1
+          {{- include "deploy.Probehealthcheck" . | nindent 10 }}
           tcpSocket:
             port: 3306
-          timeoutSeconds: 1
         startupProbe:
-          failureThreshold: 40
-          periodSeconds: 5
-          successThreshold: 1
+          {{- include "deploy.Probehealthcheck" . | nindent 10 }}
           tcpSocket:
             port: 3306
-          timeoutSeconds: 1
         {{- end }}
         {{- if .Values.develop.dev }}
         resources:
diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-logtsash.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-logtsash.yaml
index 15cdb08d7..e07f318eb 100644
--- a/deploy/kubernetes/helm/templates/deployments/dongtai-logtsash.yaml
+++ b/deploy/kubernetes/helm/templates/deployments/dongtai-logtsash.yaml
@@ -61,29 +61,17 @@ spec:
           name: log-http
         {{- if .Values.healthcheck }}
         livenessProbe:
-          failureThreshold: 3
-          initialDelaySeconds: 30
-          periodSeconds: 10
-          successThreshold: 1
+          {{- include "deploy.Probehealthcheck" . | nindent 10 }}
           tcpSocket:
             port: 9600
-          timeoutSeconds: 5
         readinessProbe:
-          failureThreshold: 3
-          initialDelaySeconds: 30
-          periodSeconds: 10
-          successThreshold: 1
+          {{- include "deploy.Probehealthcheck" . | nindent 10 }}
           tcpSocket:
             port: 9600
-          timeoutSeconds: 5
         startupProbe:
-          failureThreshold: 3
-          initialDelaySeconds: 30
-          periodSeconds: 10
-          successThreshold: 1
+          {{- include "deploy.Probehealthcheck" . | nindent 10 }}
           tcpSocket:
             port: 9600
-          timeoutSeconds: 5
         {{- end }}
         resources:
           requests:
diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-server.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-server.yaml
index 58f2f6446..faf9e0218 100644
--- a/deploy/kubernetes/helm/templates/deployments/dongtai-server.yaml
+++ b/deploy/kubernetes/helm/templates/deployments/dongtai-server.yaml
@@ -44,36 +44,24 @@ spec:
           {{- include "deploy.config" . | nindent 10 }}
           {{- if .Values.healthcheck }}
           livenessProbe:
-            failureThreshold: 3
+            {{- include "deploy.Probehealthcheck" . | nindent 12 }}
             httpGet:
               path: /healthcheck
               port: 8000
               scheme: HTTP
-            initialDelaySeconds: 30
-            periodSeconds: 10
-            successThreshold: 1
-            timeoutSeconds: 5
           name: dongtai-server-container
           readinessProbe:
-            failureThreshold: 3
+            {{- include "deploy.Probehealthcheck" . | nindent 12 }}
             httpGet:
               path: /healthcheck
               port: 8000
               scheme: HTTP
-            initialDelaySeconds: 30
-            periodSeconds: 10
-            successThreshold: 1
-            timeoutSeconds: 5
           startupProbe:
-            failureThreshold: 3
+            {{- include "deploy.Probehealthcheck" . | nindent 12 }}
             httpGet:
               path: /healthcheck
               port: 8000
               scheme: HTTP
-            initialDelaySeconds: 30
-            periodSeconds: 10
-            successThreshold: 1
-            timeoutSeconds: 5
           {{- end }}
       {{- include "deploy.config.vo" . | nindent 6 }}
       {{- if .Values.somaxconn }}
diff --git a/deploy/kubernetes/helm/values.yaml b/deploy/kubernetes/helm/values.yaml
index 59946e29c..97e5d1cc5 100644
--- a/deploy/kubernetes/helm/values.yaml
+++ b/deploy/kubernetes/helm/values.yaml
@@ -15,7 +15,7 @@ accessType: ClusterIP  # NodePort, LoadBalancer, ClusterIP
 NodePort: 30080
 logging_level: INFO # DEBUG, INFO
 somaxconn: null #If system max net.core.somaxconn (128) . Example: somaxconn: 4096
-healthcheck: false
+healthcheck: true
 
 nodeSelector:
   kubernetes.io/os: linux

From 220091b20d227e695f84d718623ce3b5c372e06e Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Thu, 13 Jul 2023 11:58:08 +0800
Subject: [PATCH 133/161] feat: update 1.13.0 strategy.

---
 dongtai_engine/common/queryset.py   |    2 +-
 static/data/go_full_policy.json     |    2 +-
 static/data/go_policy.json          |   20 -
 static/data/java_full_policy.json   |  294 +--
 static/data/java_policy.json        | 3451 +++++++++------------------
 static/data/python_full_policy.json |    4 +-
 static/data/python_policy.json      |  222 --
 static/data/vul_strategy.json       |   12 +-
 8 files changed, 1337 insertions(+), 2670 deletions(-)

diff --git a/dongtai_engine/common/queryset.py b/dongtai_engine/common/queryset.py
index c61ae6c1e..511433af8 100644
--- a/dongtai_engine/common/queryset.py
+++ b/dongtai_engine/common/queryset.py
@@ -28,7 +28,7 @@ def load_sink_strategy(user=None, language=None, scan_id=0) -> List[Dict]:
     language_id = 0
     if language and language in LANGUAGE_MAP:
         language_id = LANGUAGE_MAP[language]
-    q = ~Q(state='delete')
+    q = Q(state='enable')
     scan_template = IastStrategyUser.objects.filter(pk=scan_id).first()
     if scan_template:
         strategy_id = [int(i) for i in scan_template.content.split(',')]
diff --git a/static/data/go_full_policy.json b/static/data/go_full_policy.json
index 14a0f0206..70157f70a 100644
--- a/static/data/go_full_policy.json
+++ b/static/data/go_full_policy.json
@@ -88,7 +88,7 @@
                 "value": "template.(*Template).ExecuteTemplate()"
             }
         ],
-        "enable": 1,
+        "enable": 0,
         "type": 4,
         "value": "reflected-xss"
     },
diff --git a/static/data/go_policy.json b/static/data/go_policy.json
index 0c7a1c797..4635fa029 100644
--- a/static/data/go_policy.json
+++ b/static/data/go_policy.json
@@ -52,26 +52,6 @@
         "type": 4,
         "value": "path-traversal"
     },
-    {
-        "details": [
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P3",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "template.(*Template).ExecuteTemplate()"
-            }
-        ],
-        "enable": 1,
-        "type": 4,
-        "value": "reflected-xss"
-    },
     {
         "details": [
             {
diff --git a/static/data/java_full_policy.json b/static/data/java_full_policy.json
index 2956ba539..9973db80a 100644
--- a/static/data/java_full_policy.json
+++ b/static/data/java_full_policy.json
@@ -1041,24 +1041,6 @@
     },
     {
         "details": [
-            {
-                "command": "",
-                "created_by": 1,
-                "enable": 1,
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "language": 1,
-                "source": "P1",
-                "stack_blacklist": [],
-                "system_type": 1,
-                "tags": [],
-                "target": "O",
-                "track": "",
-                "type": 1,
-                "untags": [],
-                "value": "java.lang.String.(byte[])"
-            },
             {
                 "command": "KEEP()",
                 "created_by": 1,
@@ -1312,7 +1294,7 @@
                 "value": "java.lang.String.getBytes()"
             },
             {
-                "command": "",
+                "command": "OVERWRITE(P2)",
                 "created_by": 1,
                 "enable": 1,
                 "ignore_blacklist": false,
@@ -1581,24 +1563,6 @@
                 "untags": [],
                 "value": "java.lang.String.trim()"
             },
-            {
-                "command": "",
-                "created_by": 1,
-                "enable": 1,
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "language": 1,
-                "source": "P1",
-                "stack_blacklist": [],
-                "system_type": 1,
-                "tags": [],
-                "target": "R",
-                "track": "false",
-                "type": 1,
-                "untags": [],
-                "value": "java.lang.String.valueOf(java.lang.Object)"
-            },
             {
                 "command": "KEEP()",
                 "created_by": 1,
@@ -1635,42 +1599,6 @@
                 "untags": [],
                 "value": "java.lang.StringConcatHelper.newString(byte[],long)"
             },
-            {
-                "command": "",
-                "created_by": 1,
-                "enable": 1,
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "language": 1,
-                "source": "P4",
-                "stack_blacklist": [],
-                "system_type": 1,
-                "tags": [],
-                "target": "P2",
-                "track": "false",
-                "type": 1,
-                "untags": [],
-                "value": "java.lang.StringConcatHelper.prepend(int,byte[],byte,java.lang.String)"
-            },
-            {
-                "command": "",
-                "created_by": 1,
-                "enable": 1,
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "language": 1,
-                "source": "P4",
-                "stack_blacklist": [],
-                "system_type": 1,
-                "tags": [],
-                "target": "P2",
-                "track": "false",
-                "type": 1,
-                "untags": [],
-                "value": "java.lang.StringConcatHelper.prepend(long,byte[],byte,java.lang.String)"
-            },
             {
                 "command": "SUBSET(P2,P3)",
                 "created_by": 1,
@@ -1851,7 +1779,7 @@
                 "untags": [
                     "xml-decoded"
                 ],
-                "value": "org.thymeleaf.util.DOMUtils.escapeXml(char[], boolean)"
+                "value": "org.thymeleaf.util.DOMUtils.escapeXml(char[],boolean)"
             },
             {
                 "command": "",
@@ -1873,7 +1801,7 @@
                 "untags": [
                     "xml-decoded"
                 ],
-                "value": "org.thymeleaf.util.DOMUtils.escapeXml(java.lang.String, boolean)"
+                "value": "org.thymeleaf.util.DOMUtils.escapeXml(java.lang.String,boolean)"
             }
         ],
         "enable": 1,
@@ -4211,6 +4139,24 @@
                 "untags": [],
                 "value": "com.alibaba.fastjson.parser.DefaultJSONParser.(java.lang.String,com.alibaba.fastjson.parser.ParserConfig,int)"
             },
+            {
+                "command": "",
+                "created_by": 1,
+                "enable": 1,
+                "ignore_blacklist": false,
+                "ignore_internal": false,
+                "inherit": "all",
+                "language": 1,
+                "source": "O",
+                "stack_blacklist": [],
+                "system_type": 1,
+                "tags": [],
+                "target": "R",
+                "track": "false",
+                "type": 1,
+                "untags": [],
+                "value": "com.alibaba.fastjson.parser.JSONLexerBase.scanFieldBigInteger(char[])"
+            },
             {
                 "command": "",
                 "created_by": 1,
@@ -4683,7 +4629,7 @@
                 "value": "org.glassfish.grizzly.http.Cookie.setSecure(boolean)"
             }
         ],
-        "enable": 1,
+        "enable": 0,
         "type": 4,
         "value": "cookie-flags-missing"
     },
@@ -4750,7 +4696,7 @@
                 "value": "javax.crypto.Cipher.getInstance(java.lang.String,java.security.Provider)"
             }
         ],
-        "enable": 1,
+        "enable": 0,
         "type": 4,
         "value": "crypto-bad-ciphers"
     },
@@ -4851,7 +4797,7 @@
                 "value": "java.security.MessageDigest.getInstance(java.lang.String,java.security.Provider)"
             }
         ],
-        "enable": 1,
+        "enable": 0,
         "type": 4,
         "value": "crypto-bad-mac"
     },
@@ -4893,6 +4839,24 @@
                 "untags": [],
                 "value": "java.util.Random.nextBoolean()"
             },
+            {
+                "command": "",
+                "created_by": 1,
+                "enable": 1,
+                "ignore_blacklist": false,
+                "ignore_internal": false,
+                "inherit": "false",
+                "language": 1,
+                "source": "P1",
+                "stack_blacklist": [],
+                "system_type": 1,
+                "tags": [],
+                "target": "",
+                "track": "true",
+                "type": 4,
+                "untags": [],
+                "value": "java.util.Random.nextBytes(byte[])"
+            },
             {
                 "command": "",
                 "created_by": 1,
@@ -5123,7 +5087,7 @@
                 "value": "org.apache.commons.lang.RandomStringUtils.randomNumeric(int)"
             }
         ],
-        "enable": 1,
+        "enable": 0,
         "type": 4,
         "value": "crypto-weak-randomness"
     },
@@ -5154,6 +5118,24 @@
     },
     {
         "details": [
+            {
+                "command": "",
+                "created_by": 1,
+                "enable": 1,
+                "ignore_blacklist": false,
+                "ignore_internal": false,
+                "inherit": "false",
+                "language": 1,
+                "source": "P2",
+                "stack_blacklist": [],
+                "system_type": 1,
+                "tags": [],
+                "target": "",
+                "track": "false",
+                "type": 4,
+                "untags": [],
+                "value": "java.lang.ClassLoader.loadLibrary(java.lang.Class,java.lang.String,boolean)"
+            },
             {
                 "command": "",
                 "created_by": 1,
@@ -8987,6 +8969,24 @@
                 "type": 2,
                 "untags": [],
                 "value": "org.springframework.web.servlet.mvc.method.annotation.PathVariableMethodArgumentResolver.resolveName(java.lang.String,org.springframework.core.MethodParameter,org.springframework.web.context.request.NativeWebRequest)"
+            },
+            {
+                "command": "",
+                "created_by": 1,
+                "enable": 1,
+                "ignore_blacklist": false,
+                "ignore_internal": false,
+                "inherit": "false",
+                "language": 1,
+                "source": "O",
+                "stack_blacklist": [],
+                "system_type": 1,
+                "tags": [],
+                "target": "R",
+                "track": "false",
+                "type": 2,
+                "untags": [],
+                "value": "org.springframework.web.util.pattern.PathPattern.getPatternString()"
             }
         ],
         "enable": 1,
@@ -10786,7 +10786,7 @@
                 "value": "org.springframework.http.converter.StringHttpMessageConverter.writeInternal(java.lang.String,org.springframework.http.HttpOutputMessage)"
             }
         ],
-        "enable": 1,
+        "enable": 0,
         "type": 4,
         "value": "reflected-xss"
     },
@@ -11380,6 +11380,42 @@
                 "untags": [],
                 "value": "org.springframework.cloud.config.server.resource.ResourceController.retrieve(java.lang.String,java.lang.String,java.lang.String,java.lang.String,boolean)"
             },
+            {
+                "command": "",
+                "created_by": 1,
+                "enable": 1,
+                "ignore_blacklist": false,
+                "ignore_internal": false,
+                "inherit": "all",
+                "language": 1,
+                "source": "P1",
+                "stack_blacklist": [],
+                "system_type": 1,
+                "tags": [],
+                "target": "R",
+                "track": "false",
+                "type": 1,
+                "untags": [],
+                "value": "org.springframework.http.server.ServletServerHttpRequest.getBodyFromServletRequestParameters(jakarta.servlet.http.HttpServletRequest)"
+            },
+            {
+                "command": "",
+                "created_by": 1,
+                "enable": 1,
+                "ignore_blacklist": false,
+                "ignore_internal": false,
+                "inherit": "all",
+                "language": 1,
+                "source": "P1",
+                "stack_blacklist": [],
+                "system_type": 1,
+                "tags": [],
+                "target": "R",
+                "track": "false",
+                "type": 1,
+                "untags": [],
+                "value": "org.springframework.http.server.ServletServerHttpRequest.getBodyFromServletRequestParameters(javax.servlet.http.HttpServletRequest)"
+            },
             {
                 "command": "",
                 "created_by": 1,
@@ -12852,7 +12888,7 @@
                 "value": "play.mvc.Http$Session.put(java.lang.String,java.lang.String)"
             }
         ],
-        "enable": 1,
+        "enable": 0,
         "type": 4,
         "value": "trust-boundary-violation"
     },
@@ -14437,6 +14473,24 @@
                 "untags": [],
                 "value": "jakarta.xml.bind.Unmarshaller.unmarshal(org.xml.sax.InputSource)"
             },
+            {
+                "command": "",
+                "created_by": 1,
+                "enable": 1,
+                "ignore_blacklist": false,
+                "ignore_internal": false,
+                "inherit": "true",
+                "language": 1,
+                "source": "P1",
+                "stack_blacklist": [],
+                "system_type": 1,
+                "tags": [],
+                "target": "",
+                "track": "true",
+                "type": 4,
+                "untags": [],
+                "value": "jakarta.xml.parsers.DocumentBuilder.parse(java.io.File)"
+            },
             {
                 "command": "",
                 "created_by": 1,
@@ -15122,31 +15176,6 @@
                 "type": 4,
                 "untags": [],
                 "value": "nu.xom.Builder.build(java.lang.String)"
-            }
-        ],
-        "enable": 1,
-        "type": 4,
-        "value": "xxe"
-    },
-    {
-        "details": [
-            {
-                "command": "",
-                "created_by": 1,
-                "enable": 1,
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "language": 1,
-                "source": "",
-                "stack_blacklist": [],
-                "system_type": 1,
-                "tags": [],
-                "target": "",
-                "track": "",
-                "type": 3,
-                "untags": [],
-                "value": "com.sun.org.apache.xerces.internal.xni.grammars.XMLGrammarLoader.setFeature(java.lang.String, boolean)"
             },
             {
                 "command": "",
@@ -15156,16 +15185,23 @@
                 "ignore_internal": false,
                 "inherit": "true",
                 "language": 1,
-                "source": "",
+                "source": "P1",
                 "stack_blacklist": [],
                 "system_type": 1,
                 "tags": [],
                 "target": "",
-                "track": "",
-                "type": 3,
+                "track": "true",
+                "type": 4,
                 "untags": [],
-                "value": "com.sun.org.apache.xerces.internal.xni.parser.XMLParserConfiguration.setFeature(java.lang.String, boolean)"
-            },
+                "value": "org.xml.sax.XMLReader.parse(org.xml.sax.InputSource)"
+            }
+        ],
+        "enable": 1,
+        "type": 4,
+        "value": "xxe"
+    },
+    {
+        "details": [
             {
                 "command": "",
                 "created_by": 1,
@@ -15201,42 +15237,6 @@
                 "type": 3,
                 "untags": [],
                 "value": "javax.xml.parsers.SAXParserFactory.setFeature(java.lang.String, boolean)"
-            },
-            {
-                "command": "",
-                "created_by": 1,
-                "enable": 1,
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "language": 1,
-                "source": "",
-                "stack_blacklist": [],
-                "system_type": 1,
-                "tags": [],
-                "target": "",
-                "track": "",
-                "type": 3,
-                "untags": [],
-                "value": "javax.xml.transform.TransformerFactory.setFeature(java.lang.String, boolean)"
-            },
-            {
-                "command": "",
-                "created_by": 1,
-                "enable": 1,
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "language": 1,
-                "source": "",
-                "stack_blacklist": [],
-                "system_type": 1,
-                "tags": [],
-                "target": "",
-                "track": "",
-                "type": 3,
-                "untags": [],
-                "value": "org.xml.sax.XMLReader.setFeature(java.lang.String, boolean)"
             }
         ],
         "enable": 1,
diff --git a/static/data/java_policy.json b/static/data/java_policy.json
index 73b4ac427..6b58a26a9 100644
--- a/static/data/java_policy.json
+++ b/static/data/java_policy.json
@@ -791,19 +791,6 @@
     },
     {
         "details": [
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "O",
-                "track": "",
-                "untags": [],
-                "value": "java.lang.String.(byte[])"
-            },
             {
                 "command": "KEEP()",
                 "ignore_blacklist": false,
@@ -987,7 +974,7 @@
                 "value": "java.lang.String.getBytes()"
             },
             {
-                "command": "",
+                "command": "OVERWRITE(P2)",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
@@ -1181,19 +1168,6 @@
                 "untags": [],
                 "value": "java.lang.String.trim()"
             },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "R",
-                "track": "false",
-                "untags": [],
-                "value": "java.lang.String.valueOf(java.lang.Object)"
-            },
             {
                 "command": "KEEP()",
                 "ignore_blacklist": false,
@@ -1220,32 +1194,6 @@
                 "untags": [],
                 "value": "java.lang.StringConcatHelper.newString(byte[],long)"
             },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P4",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "P2",
-                "track": "false",
-                "untags": [],
-                "value": "java.lang.StringConcatHelper.prepend(int,byte[],byte,java.lang.String)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P4",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "P2",
-                "track": "false",
-                "untags": [],
-                "value": "java.lang.StringConcatHelper.prepend(long,byte[],byte,java.lang.String)"
-            },
             {
                 "command": "SUBSET(P2,P3)",
                 "ignore_blacklist": false,
@@ -1381,7 +1329,7 @@
                 "untags": [
                     "xml-decoded"
                 ],
-                "value": "org.thymeleaf.util.DOMUtils.escapeXml(char[], boolean)"
+                "value": "org.thymeleaf.util.DOMUtils.escapeXml(char[],boolean)"
             },
             {
                 "command": "",
@@ -1398,7 +1346,7 @@
                 "untags": [
                     "xml-decoded"
                 ],
-                "value": "org.thymeleaf.util.DOMUtils.escapeXml(java.lang.String, boolean)"
+                "value": "org.thymeleaf.util.DOMUtils.escapeXml(java.lang.String,boolean)"
             }
         ],
         "enable": 1,
@@ -3136,6 +3084,19 @@
                 "untags": [],
                 "value": "com.alibaba.fastjson.parser.DefaultJSONParser.(java.lang.String,com.alibaba.fastjson.parser.ParserConfig,int)"
             },
+            {
+                "command": "",
+                "ignore_blacklist": false,
+                "ignore_internal": false,
+                "inherit": "all",
+                "source": "O",
+                "stack_blacklist": [],
+                "tags": [],
+                "target": "R",
+                "track": "false",
+                "untags": [],
+                "value": "com.alibaba.fastjson.parser.JSONLexerBase.scanFieldBigInteger(char[])"
+            },
             {
                 "command": "",
                 "ignore_blacklist": false,
@@ -3430,62 +3391,56 @@
     },
     {
         "details": [
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "javax.servlet.http.Cookie.setSecure(boolean)"
-            },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
-                "source": "P9",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "",
+                "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "javax.ws.rs.core.NewCookie.(java.lang.String,java.lang.String,java.lang.String,java.lang.String,int,java.lang.String,int,java.util.Date,boolean,boolean)"
-            },
+                "value": "org.iast.springsec.common.DataManager.doManage(java.lang.String)"
+            }
+        ],
+        "enable": 1,
+        "type": 1,
+        "value": "custom-encrypt"
+    },
+    {
+        "details": [
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
-                "source": "P5",
+                "source": "P2",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "false",
                 "untags": [],
-                "value": "javax.ws.rs.core.NewCookie.(javax.ws.rs.core.Cookie,java.lang.String,int,java.util.Date,boolean,boolean)"
+                "value": "java.lang.ClassLoader.loadLibrary(java.lang.Class,java.lang.String,boolean)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
-                "source": "P1",
+                "source": "P2",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
-                "track": "false",
+                "track": "true",
                 "untags": [],
-                "value": "org.glassfish.grizzly.http.Cookie.setSecure(boolean)"
+                "value": "java.lang.Runtime.load0(java.lang.Class,java.lang.String)"
             }
         ],
         "enable": 1,
         "type": 4,
-        "value": "cookie-flags-missing"
+        "value": "dynamic-library-load"
     },
     {
         "details": [
@@ -3493,382 +3448,288 @@
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "all",
                 "source": "P1",
-                "stack_blacklist": [
-                    "com.ibm.ejs.util.am._Alarm.run",
-                    "com.ibm.crypto.provider.PKCS12KeyStore.engineLoad",
-                    "util.StateUtils.encrypt"
-                ],
+                "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "javax.crypto.Cipher.getInstance(java.lang.String)"
+                "value": "jakarta.el.ELProcessor.eval(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "all",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
-                "track": "false",
+                "track": "true",
                 "untags": [],
-                "value": "javax.crypto.Cipher.getInstance(java.lang.String,java.lang.String)"
+                "value": "jakarta.el.ELProcessor.getValue(java.lang.String,java.lang.Class)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "all",
                 "source": "P1",
-                "stack_blacklist": [
-                    "com.ca.siteminder"
-                ],
+                "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "javax.crypto.Cipher.getInstance(java.lang.String,java.security.Provider)"
-            }
-        ],
-        "enable": 1,
-        "type": 4,
-        "value": "crypto-bad-ciphers"
-    },
-    {
-        "details": [
+                "value": "jakarta.el.ELProcessor.setValue(java.lang.String,java.lang.Object)"
+            },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
-                "stack_blacklist": [
-                    "com.mysql.jdbc",
-                    "org.skife.jdbi.v2.Query",
-                    "com.amazonaws.services.s3.AmazonS3Client.putObject",
-                    "com.ibm.crypto.provider.PKCS12KeyStore.engineLoad",
-                    "com.ibm.ws.security.ltpa.LTPAToken2.getBytes",
-                    "com.ibm.ws.ssl.channel.impl.SSLUtils.handleHandshake",
-                    "com.jcraft.jsch.Session.connect",
-                    "com.microsoft.sqlserver.jdbc.TDSChannel.enableSS",
-                    "com.newrelic.agent",
-                    "com.compuware.apm.agent",
-                    "asset.pipeline.AssetPipeline.serveUncompiledAsset",
-                    "controllers.AssetsBuilder",
-                    "JITCompiler",
-                    "java.security.SecureRandom",
-                    "java.util.jar.JarVerifier",
-                    "javax.crypto.JarVerifier",
-                    "jakarta.crypto.JarVerifier",
-                    "maybeNotModified",
-                    "oracle.jdbc.driver",
-                    "java.security.Signature.initVerify",
-                    "oracle.jdbc.xa.client.OracleXADataSource.getXAConnection",
-                    "org.eclipse.jetty.io.ssl.SslConnection",
-                    "org.springframework.web.client.RestTemplate",
-                    "org.thymeleaf.spring4.view.ThymeleafView.render",
-                    "play.api.libs.Codecs$",
-                    "play.api.mvc.CookieBaker",
-                    "play.router.RoutesCompiler",
-                    "play.PlaySourceGenerators",
-                    "sbt.compiler",
-                    "sbt.inc.Stamp",
-                    "org.jets3t.service.utils.ServiceUtils.signWithHmacSha1",
-                    "org.jboss.resteasy.spi.ResteasyDeployment.start"
-                ],
+                "inherit": "all",
+                "source": "P2",
+                "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "java.security.MessageDigest.getInstance(java.lang.String)"
+                "value": "jakarta.el.ELProcessor.setVariable(java.lang.String,java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
-                "stack_blacklist": [
-                    "java.security.SecureRandom",
-                    "java.util.jar.JarVerifier",
-                    "com.microsoft.sqlserver.jdbc.TDSChannel.enableSS"
-                ],
+                "inherit": "all",
+                "source": "P2",
+                "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "java.security.MessageDigest.getInstance(java.lang.String,java.lang.String)"
+                "value": "jakarta.el.ExpressionFactory.createMethodExpression(jakarta.el.ELContext,java.lang.String,java.lang.Class,java.lang.Class[])"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
-                "stack_blacklist": [
-                    "java.security.SecureRandom",
-                    "java.util.jar.JarVerifier"
-                ],
+                "inherit": "all",
+                "source": "P2",
+                "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "java.security.MessageDigest.getInstance(java.lang.String,java.security.Provider)"
-            }
-        ],
-        "enable": 1,
-        "type": 4,
-        "value": "crypto-bad-mac"
-    },
-    {
-        "details": [
+                "value": "jakarta.el.ExpressionFactory.createValueExpression(jakarta.el.ELContext,java.lang.String,java.lang.Class)"
+            },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "",
+                "inherit": "all",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
-                "track": "false",
+                "track": "true",
                 "untags": [],
-                "value": "java.lang.Math.random()"
+                "value": "jakarta.servlet.jsp.el.ExpressionEvaluator.evaluate(java.lang.String,java.lang.Class,jakarta.servlet.jsp.el.VariableResolver,jakarta.servlet.jsp.el.FunctionMapper)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "",
+                "inherit": "all",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
-                "track": "false",
+                "track": "true",
                 "untags": [],
-                "value": "java.util.Random.nextBoolean()"
+                "value": "javax.el.ELProcessor.eval(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "",
+                "inherit": "all",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
-                "track": "false",
+                "track": "true",
                 "untags": [],
-                "value": "java.util.Random.nextDouble()"
+                "value": "javax.el.ELProcessor.getValue(java.lang.String,java.lang.Class)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "O",
-                "stack_blacklist": [
-                    "weblogic.work.IncrementAdvisor.run"
-                ],
+                "inherit": "all",
+                "source": "P1",
+                "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "java.util.Random.nextFloat()"
+                "value": "javax.el.ELProcessor.setValue(java.lang.String,java.lang.Object)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "",
+                "inherit": "all",
+                "source": "P2",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
-                "track": "false",
+                "track": "true",
                 "untags": [],
-                "value": "java.util.Random.nextGaussian()"
+                "value": "javax.el.ELProcessor.setVariable(java.lang.String,java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "O",
-                "stack_blacklist": [
-                    "com.google.gson.JsonObject",
-                    "java.util.Hashtable"
-                ],
+                "inherit": "all",
+                "source": "P2",
+                "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "java.util.Random.nextInt()"
+                "value": "javax.el.ExpressionFactory.createMethodExpression(javax.el.ELContext,java.lang.String,java.lang.Class,java.lang.Class[])"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "O",
-                "stack_blacklist": [
-                    "getRandomSample",
-                    "java.util.Hashtable",
-                    "NullSafeConcurrentHashMap",
-                    "org.apache.tomcat.websocket.WsWebSocketContainer.generateWsKeyValue",
-                    "org.quartz.core.QuartzSchedulerThread.getRandomizedIdleWaitTime",
-                    "SelectableConcurrentHashMap",
-                    "net.bytebuddy.utility.RandomString.nextString"
-                ],
+                "inherit": "all",
+                "source": "P2",
+                "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "java.util.Random.nextInt(int)"
+                "value": "javax.el.ExpressionFactory.createValueExpression(javax.el.ELContext,java.lang.String,java.lang.Class)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "",
+                "inherit": "all",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
-                "track": "false",
+                "track": "true",
                 "untags": [],
-                "value": "java.util.Random.nextLong()"
+                "value": "javax.servlet.jsp.el.ExpressionEvaluator.evaluate(java.lang.String,java.lang.Class,javax.servlet.jsp.el.VariableResolver,javax.servlet.jsp.el.FunctionMapper)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "",
+                "inherit": "all",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
-                "track": "false",
+                "track": "true",
                 "untags": [],
-                "value": "org.apache.commons.lang.RandomStringUtils.random(int,int,int,boolean,boolean)"
+                "value": "ognl.Ognl.getValue(java.lang.Object,java.lang.Object)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "",
+                "inherit": "all",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
-                "track": "false",
+                "track": "true",
                 "untags": [],
-                "value": "org.apache.commons.lang.RandomStringUtils.random(int,int,int,boolean,boolean,char[])"
+                "value": "ognl.Ognl.getValue(java.lang.Object,java.lang.Object,java.lang.Class)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "",
+                "inherit": "all",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
-                "track": "false",
+                "track": "true",
                 "untags": [],
-                "value": "org.apache.commons.lang.RandomStringUtils.randomAlphabetic(int)"
+                "value": "ognl.Ognl.getValue(java.lang.Object,java.util.Map,java.lang.Object)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "",
+                "inherit": "all",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
-                "track": "false",
+                "track": "true",
                 "untags": [],
-                "value": "org.apache.commons.lang.RandomStringUtils.randomAlphanumeric(int)"
+                "value": "ognl.Ognl.getValue(java.lang.Object,java.util.Map,java.lang.Object,java.lang.Class)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "",
+                "inherit": "all",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
-                "track": "false",
+                "track": "true",
                 "untags": [],
-                "value": "org.apache.commons.lang.RandomStringUtils.randomAscii(int)"
+                "value": "ognl.Ognl.getValue(java.lang.String,java.lang.Object)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "",
+                "inherit": "all",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
-                "track": "false",
+                "track": "true",
                 "untags": [],
-                "value": "org.apache.commons.lang.RandomStringUtils.randomNumeric(int)"
-            }
-        ],
-        "enable": 1,
-        "type": 4,
-        "value": "crypto-weak-randomness"
-    },
-    {
-        "details": [
+                "value": "ognl.Ognl.getValue(java.lang.String,java.lang.Object,java.lang.Class)"
+            },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "all",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "R",
-                "track": "false",
+                "target": "",
+                "track": "true",
                 "untags": [],
-                "value": "org.iast.springsec.common.DataManager.doManage(java.lang.String)"
-            }
-        ],
-        "enable": 1,
-        "type": 1,
-        "value": "custom-encrypt"
-    },
-    {
-        "details": [
+                "value": "ognl.Ognl.getValue(java.lang.String,java.util.Map,java.lang.Object)"
+            },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "P2",
+                "inherit": "all",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "java.lang.Runtime.load0(java.lang.Class,java.lang.String)"
-            }
-        ],
-        "enable": 1,
-        "type": 4,
-        "value": "dynamic-library-load"
-    },
-    {
-        "details": [
+                "value": "ognl.Ognl.getValue(java.lang.String,java.util.Map,java.lang.Object,java.lang.Class)"
+            },
             {
                 "command": "",
                 "ignore_blacklist": false,
@@ -3880,7 +3741,7 @@
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "jakarta.el.ELProcessor.eval(java.lang.String)"
+                "value": "ognl.Ognl.parseExpression(java.lang.String)"
             },
             {
                 "command": "",
@@ -3893,163 +3754,163 @@
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "jakarta.el.ELProcessor.getValue(java.lang.String,java.lang.Class)"
+                "value": "org.apache.commons.ognl.Ognl.parseExpression(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
+                "inherit": "true",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "jakarta.el.ELProcessor.setValue(java.lang.String,java.lang.Object)"
+                "value": "org.springframework.expression.Expression.getValue()"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "all",
-                "source": "P2",
+                "inherit": "true",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "jakarta.el.ELProcessor.setVariable(java.lang.String,java.lang.String)"
+                "value": "org.springframework.expression.Expression.getValue(java.lang.Class)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "all",
-                "source": "P2",
+                "inherit": "true",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "jakarta.el.ExpressionFactory.createMethodExpression(jakarta.el.ELContext,java.lang.String,java.lang.Class,java.lang.Class[])"
+                "value": "org.springframework.expression.Expression.getValue(java.lang.Object)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "all",
-                "source": "P2",
+                "inherit": "true",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "jakarta.el.ExpressionFactory.createValueExpression(jakarta.el.ELContext,java.lang.String,java.lang.Class)"
+                "value": "org.springframework.expression.Expression.getValue(java.lang.Object,java.lang.Class)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
+                "inherit": "true",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "jakarta.servlet.jsp.el.ExpressionEvaluator.evaluate(java.lang.String,java.lang.Class,jakarta.servlet.jsp.el.VariableResolver,jakarta.servlet.jsp.el.FunctionMapper)"
+                "value": "org.springframework.expression.Expression.getValue(org.springframework.expression.EvaluationContext)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
+                "inherit": "true",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "javax.el.ELProcessor.eval(java.lang.String)"
+                "value": "org.springframework.expression.Expression.getValue(org.springframework.expression.EvaluationContext,java.lang.Class)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
+                "inherit": "true",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "javax.el.ELProcessor.getValue(java.lang.String,java.lang.Class)"
+                "value": "org.springframework.expression.Expression.getValue(org.springframework.expression.EvaluationContext,java.lang.Object)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
+                "inherit": "true",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "javax.el.ELProcessor.setValue(java.lang.String,java.lang.Object)"
+                "value": "org.springframework.expression.Expression.getValue(org.springframework.expression.EvaluationContext,java.lang.Object,java.lang.Class)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "all",
-                "source": "P2",
+                "inherit": "true",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "javax.el.ELProcessor.setVariable(java.lang.String,java.lang.String)"
+                "value": "org.springframework.expression.Expression.getValueTypeDescriptor()"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "all",
-                "source": "P2",
+                "inherit": "true",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "javax.el.ExpressionFactory.createMethodExpression(javax.el.ELContext,java.lang.String,java.lang.Class,java.lang.Class[])"
+                "value": "org.springframework.expression.Expression.getValueTypeDescriptor(java.lang.Object)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "all",
-                "source": "P2",
+                "inherit": "true",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "javax.el.ExpressionFactory.createValueExpression(javax.el.ELContext,java.lang.String,java.lang.Class)"
+                "value": "org.springframework.expression.Expression.getValueTypeDescriptor(org.springframework.expression.EvaluationContext)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
+                "inherit": "true",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "javax.servlet.jsp.el.ExpressionEvaluator.evaluate(java.lang.String,java.lang.Class,javax.servlet.jsp.el.VariableResolver,javax.servlet.jsp.el.FunctionMapper)"
+                "value": "org.springframework.expression.Expression.getValueTypeDescriptor(org.springframework.expression.EvaluationContext,java.lang.Object)"
             },
             {
                 "command": "",
@@ -4062,372 +3923,372 @@
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "ognl.Ognl.getValue(java.lang.Object,java.lang.Object)"
+                "value": "org.thymeleaf.standard.expression.Expression.parse(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
+                "inherit": "true",
+                "source": "P2",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "ognl.Ognl.getValue(java.lang.Object,java.lang.Object,java.lang.Class)"
+                "value": "org.thymeleaf.standard.expression.IStandardExpressionParser.parseExpression(org.thymeleaf.context.IExpressionContext,java.lang.String)"
+            }
+        ],
+        "enable": 1,
+        "type": 4,
+        "value": "expression-language-injection"
+    },
+    {
+        "details": [
+            {
+                "command": "",
+                "ignore_blacklist": false,
+                "ignore_internal": false,
+                "inherit": "true",
+                "source": "O",
+                "stack_blacklist": [],
+                "tags": [],
+                "target": "R",
+                "track": "false",
+                "untags": [],
+                "value": "org.apache.commons.fileupload.FileItem.getName()"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
+                "inherit": "true",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "",
-                "track": "true",
+                "target": "R",
+                "track": "false",
                 "untags": [],
-                "value": "ognl.Ognl.getValue(java.lang.Object,java.util.Map,java.lang.Object)"
+                "value": "org.springframework.web.multipart.MultipartFile.getName()"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
+                "inherit": "true",
+                "source": "O",
+                "stack_blacklist": [],
+                "tags": [],
+                "target": "R",
+                "track": "false",
+                "untags": [],
+                "value": "org.springframework.web.multipart.MultipartFile.getOriginalFilename()"
+            }
+        ],
+        "enable": 1,
+        "type": 1,
+        "value": "fileupload"
+    },
+    {
+        "details": [
+            {
+                "command": "",
+                "ignore_blacklist": false,
+                "ignore_internal": false,
+                "inherit": "true",
+                "source": "P2",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "ognl.Ognl.getValue(java.lang.Object,java.util.Map,java.lang.Object,java.lang.Class)"
+                "value": "org.hibernate.Session.createFilter(java.lang.Object,java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "all",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "ognl.Ognl.getValue(java.lang.String,java.lang.Object)"
+                "value": "org.hibernate.Session.createQuery(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "all",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "ognl.Ognl.getValue(java.lang.String,java.lang.Object,java.lang.Class)"
+                "value": "org.hibernate.Session.createSQLQuery(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "all",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "ognl.Ognl.getValue(java.lang.String,java.util.Map,java.lang.Object)"
+                "value": "org.hibernate.Session.createSQLQuery(java.lang.String,java.lang.String,java.lang.Class)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "all",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "ognl.Ognl.getValue(java.lang.String,java.util.Map,java.lang.Object,java.lang.Class)"
+                "value": "org.hibernate.Session.createSQLQuery(java.lang.String,java.lang.String[],java.lang.Class[])"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "all",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "ognl.Ognl.parseExpression(java.lang.String)"
+                "value": "org.hibernate.Session.delete(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "all",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.apache.commons.ognl.Ognl.parseExpression(java.lang.String)"
+                "value": "org.hibernate.Session.delete(java.lang.String,java.lang.Object,org.hibernate.type.Type)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "O",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.springframework.expression.Expression.getValue()"
+                "value": "org.hibernate.Session.delete(java.lang.String,java.lang.Object[],org.hibernate.type.Type[])"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "O",
+                "source": "P2",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.springframework.expression.Expression.getValue(java.lang.Class)"
+                "value": "org.hibernate.Session.filter(java.lang.Object,java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "O",
+                "source": "P2",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.springframework.expression.Expression.getValue(java.lang.Object)"
+                "value": "org.hibernate.Session.filter(java.lang.Object,java.lang.String,java.lang.Object,org.hibernate.type.Type)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "O",
+                "source": "P2",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.springframework.expression.Expression.getValue(java.lang.Object,java.lang.Class)"
+                "value": "org.hibernate.Session.filter(java.lang.Object,java.lang.String,java.lang.Object[],org.hibernate.type.Type[])"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "O",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.springframework.expression.Expression.getValue(org.springframework.expression.EvaluationContext)"
+                "value": "org.hibernate.Session.find(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "O",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.springframework.expression.Expression.getValue(org.springframework.expression.EvaluationContext,java.lang.Class)"
+                "value": "org.hibernate.Session.find(java.lang.String,java.lang.Object,org.hibernate.type.Type)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "O",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.springframework.expression.Expression.getValue(org.springframework.expression.EvaluationContext,java.lang.Object)"
+                "value": "org.hibernate.Session.find(java.lang.String,java.lang.Object[],org.hibernate.type.Type[])"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "O",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.springframework.expression.Expression.getValue(org.springframework.expression.EvaluationContext,java.lang.Object,java.lang.Class)"
+                "value": "org.hibernate.Session.iterate(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "O",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.springframework.expression.Expression.getValueTypeDescriptor()"
+                "value": "org.hibernate.Session.iterate(java.lang.String,java.lang.Object,org.hibernate.type.Type)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "O",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.springframework.expression.Expression.getValueTypeDescriptor(java.lang.Object)"
+                "value": "org.hibernate.Session.iterate(java.lang.String,java.lang.Object[],org.hibernate.type.Type[])"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "O",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.springframework.expression.Expression.getValueTypeDescriptor(org.springframework.expression.EvaluationContext)"
+                "value": "org.hibernate.SharedSessionContract.createQuery(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "O",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.springframework.expression.Expression.getValueTypeDescriptor(org.springframework.expression.EvaluationContext,java.lang.Object)"
+                "value": "org.hibernate.SharedSessionContract.createSQLQuery(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "all",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.thymeleaf.standard.expression.Expression.parse(java.lang.String)"
+                "value": "org.hibernate.criterion.Expression.sql(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "P2",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.thymeleaf.standard.expression.IStandardExpressionParser.parseExpression(org.thymeleaf.context.IExpressionContext,java.lang.String)"
-            }
-        ],
-        "enable": 1,
-        "type": 4,
-        "value": "expression-language-injection"
-    },
-    {
-        "details": [
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "source": "O",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "R",
-                "track": "false",
-                "untags": [],
-                "value": "org.apache.commons.fileupload.FileItem.getName()"
+                "value": "org.hibernate.criterion.Expression.sql(java.lang.String,java.lang.Object[],org.hibernate.type.Type[])"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "O",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "R",
-                "track": "false",
+                "target": "",
+                "track": "true",
                 "untags": [],
-                "value": "org.springframework.web.multipart.MultipartFile.getName()"
+                "value": "org.hibernate.criterion.Restrictions.sqlRestriction(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "O",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "R",
-                "track": "false",
-                "untags": [],
-                "value": "org.springframework.web.multipart.MultipartFile.getOriginalFilename()"
-            }
-        ],
-        "enable": 1,
-        "type": 1,
-        "value": "fileupload"
-    },
-    {
-        "details": [
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "source": "P2",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.hibernate.Session.createFilter(java.lang.Object,java.lang.String)"
+                "value": "org.hibernate.criterion.Restrictions.sqlRestriction(java.lang.String,java.lang.Object,org.hibernate.type.Type)"
             },
             {
                 "command": "",
@@ -4440,7 +4301,7 @@
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.hibernate.Session.createQuery(java.lang.String)"
+                "value": "org.hibernate.criterion.Restrictions.sqlRestriction(java.lang.String,java.lang.Object[],org.hibernate.type.Type[])"
             },
             {
                 "command": "",
@@ -4453,7 +4314,7 @@
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.hibernate.Session.createSQLQuery(java.lang.String)"
+                "value": "org.hibernate.query.QueryProducer.createNativeQuery(java.lang.String)"
             },
             {
                 "command": "",
@@ -4466,7 +4327,7 @@
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.hibernate.Session.createSQLQuery(java.lang.String,java.lang.String,java.lang.Class)"
+                "value": "org.hibernate.query.QueryProducer.createNativeQuery(java.lang.String,java.lang.Class)"
             },
             {
                 "command": "",
@@ -4479,303 +4340,304 @@
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.hibernate.Session.createSQLQuery(java.lang.String,java.lang.String[],java.lang.Class[])"
-            },
+                "value": "org.hibernate.query.QueryProducer.createNativeQuery(java.lang.String,java.lang.String)"
+            }
+        ],
+        "enable": 1,
+        "type": 4,
+        "value": "hql-injection"
+    },
+    {
+        "details": [
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
+                "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "",
-                "track": "true",
+                "target": "O",
+                "track": "false",
                 "untags": [],
-                "value": "org.hibernate.Session.delete(java.lang.String)"
+                "value": "org.apache.http.entity.ByteArrayEntity.(byte[],int,int,org.apache.http.entity.ContentType)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
+                "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "",
-                "track": "true",
+                "target": "O",
+                "track": "false",
                 "untags": [],
-                "value": "org.hibernate.Session.delete(java.lang.String,java.lang.Object,org.hibernate.type.Type)"
+                "value": "org.apache.http.entity.ByteArrayEntity.(byte[],org.apache.http.entity.ContentType)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
+                "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "",
-                "track": "true",
+                "target": "O",
+                "track": "false",
                 "untags": [],
-                "value": "org.hibernate.Session.delete(java.lang.String,java.lang.Object[],org.hibernate.type.Type[])"
+                "value": "org.apache.http.entity.InputStreamEntity.(java.io.InputStream,long,org.apache.http.entity.ContentType)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
-                "source": "P2",
+                "inherit": "false",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "",
-                "track": "true",
+                "target": "O",
+                "track": "false",
                 "untags": [],
-                "value": "org.hibernate.Session.filter(java.lang.Object,java.lang.String)"
+                "value": "org.apache.http.entity.StringEntity.(java.lang.String,java.lang.String,java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
-                "source": "P2",
+                "inherit": "false",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "",
-                "track": "true",
+                "target": "O",
+                "track": "false",
                 "untags": [],
-                "value": "org.hibernate.Session.filter(java.lang.Object,java.lang.String,java.lang.Object,org.hibernate.type.Type)"
-            },
+                "value": "org.apache.http.entity.StringEntity.(java.lang.String,org.apache.http.entity.ContentType)"
+            }
+        ],
+        "enable": 1,
+        "type": 1,
+        "value": "httpclient"
+    },
+    {
+        "details": [
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
-                "source": "P2",
+                "inherit": "false",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "",
-                "track": "true",
+                "target": "O",
+                "track": "false",
                 "untags": [],
-                "value": "org.hibernate.Session.filter(java.lang.Object,java.lang.String,java.lang.Object[],org.hibernate.type.Type[])"
+                "value": "org.apache.hc.core5.http.io.entity.ByteArrayEntity.(byte[],int,int,org.apache.hc.core5.http.ContentType,java.lang.String,boolean)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
+                "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "",
-                "track": "true",
+                "target": "O",
+                "track": "false",
                 "untags": [],
-                "value": "org.hibernate.Session.find(java.lang.String)"
+                "value": "org.apache.hc.core5.http.io.entity.ByteArrayEntity.(byte[],org.apache.hc.core5.http.ContentType,java.lang.String,boolean)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
+                "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "",
-                "track": "true",
+                "target": "O",
+                "track": "false",
                 "untags": [],
-                "value": "org.hibernate.Session.find(java.lang.String,java.lang.Object,org.hibernate.type.Type)"
+                "value": "org.apache.hc.core5.http.io.entity.InputStreamEntity.(java.io.InputStream,long,org.apache.hc.core5.http.ContentType,java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
+                "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "",
-                "track": "true",
+                "target": "O",
+                "track": "false",
                 "untags": [],
-                "value": "org.hibernate.Session.find(java.lang.String,java.lang.Object[],org.hibernate.type.Type[])"
-            },
+                "value": "org.apache.hc.core5.http.io.entity.StringEntity.(java.lang.String,org.apache.hc.core5.http.ContentType,java.lang.String,boolean)"
+            }
+        ],
+        "enable": 1,
+        "type": 1,
+        "value": "httpclient5"
+    },
+    {
+        "details": [
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
+                "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "",
-                "track": "true",
+                "target": "O",
+                "track": "",
                 "untags": [],
-                "value": "org.hibernate.Session.iterate(java.lang.String)"
+                "value": "java.io.BufferedReader.(java.io.InputStreamReader)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
+                "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "",
-                "track": "true",
+                "target": "O",
+                "track": "",
                 "untags": [],
-                "value": "org.hibernate.Session.iterate(java.lang.String,java.lang.Object,org.hibernate.type.Type)"
+                "value": "java.io.BufferedReader.(java.io.Reader)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
+                "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "",
-                "track": "true",
+                "target": "O",
+                "track": "",
                 "untags": [],
-                "value": "org.hibernate.Session.iterate(java.lang.String,java.lang.Object[],org.hibernate.type.Type[])"
+                "value": "java.io.BufferedReader.(java.io.Reader,int)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
-                "source": "P1",
+                "inherit": "false",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "",
-                "track": "true",
+                "target": "R",
+                "track": "",
                 "untags": [],
-                "value": "org.hibernate.SharedSessionContract.createQuery(java.lang.String)"
+                "value": "java.io.BufferedReader.readLine()"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
+                "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "",
-                "track": "true",
+                "target": "O",
+                "track": "",
                 "untags": [],
-                "value": "org.hibernate.SharedSessionContract.createSQLQuery(java.lang.String)"
+                "value": "java.io.ByteArrayInputStream.(byte[])"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
+                "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "",
-                "track": "true",
+                "target": "O",
+                "track": "",
                 "untags": [],
-                "value": "org.hibernate.criterion.Expression.sql(java.lang.String)"
+                "value": "java.io.ByteArrayInputStream.(byte[],int,int)"
             },
             {
-                "command": "",
+                "command": "REMOVE()",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
-                "source": "P1",
+                "inherit": "false",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "",
-                "track": "true",
+                "target": "O",
+                "track": "",
                 "untags": [],
-                "value": "org.hibernate.criterion.Expression.sql(java.lang.String,java.lang.Object[],org.hibernate.type.Type[])"
+                "value": "java.io.ByteArrayOutputStream.reset()"
             },
             {
-                "command": "",
+                "command": "KEEP()",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
-                "source": "P1",
+                "inherit": "false",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "",
-                "track": "true",
+                "target": "R",
+                "track": "false",
                 "untags": [],
-                "value": "org.hibernate.criterion.Restrictions.sqlRestriction(java.lang.String)"
+                "value": "java.io.ByteArrayOutputStream.toByteArray()"
             },
             {
-                "command": "",
+                "command": "KEEP()",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
-                "source": "P1",
+                "inherit": "false",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "",
-                "track": "true",
+                "target": "R",
+                "track": "false",
                 "untags": [],
-                "value": "org.hibernate.criterion.Restrictions.sqlRestriction(java.lang.String,java.lang.Object,org.hibernate.type.Type)"
+                "value": "java.io.ByteArrayOutputStream.toString()"
             },
             {
-                "command": "",
+                "command": "KEEP()",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
-                "source": "P1",
+                "inherit": "false",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "",
-                "track": "true",
+                "target": "R",
+                "track": "false",
                 "untags": [],
-                "value": "org.hibernate.criterion.Restrictions.sqlRestriction(java.lang.String,java.lang.Object[],org.hibernate.type.Type[])"
+                "value": "java.io.ByteArrayOutputStream.toString(int)"
             },
             {
-                "command": "",
+                "command": "KEEP()",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
-                "source": "P1",
+                "inherit": "false",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "",
-                "track": "true",
+                "target": "R",
+                "track": "false",
                 "untags": [],
-                "value": "org.hibernate.query.QueryProducer.createNativeQuery(java.lang.String)"
+                "value": "java.io.ByteArrayOutputStream.toString(java.lang.String)"
             },
             {
-                "command": "",
+                "command": "KEEP()",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
-                "source": "P1",
+                "inherit": "false",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "",
-                "track": "true",
+                "target": "R",
+                "track": "false",
                 "untags": [],
-                "value": "org.hibernate.query.QueryProducer.createNativeQuery(java.lang.String,java.lang.Class)"
+                "value": "java.io.ByteArrayOutputStream.toString(java.nio.charset.Charset)"
             },
             {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "org.hibernate.query.QueryProducer.createNativeQuery(java.lang.String,java.lang.String)"
-            }
-        ],
-        "enable": 1,
-        "type": 4,
-        "value": "hql-injection"
-    },
-    {
-        "details": [
-            {
-                "command": "",
+                "command": "APPEND(P2,P3)",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
@@ -4783,9 +4645,9 @@
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "O",
-                "track": "false",
+                "track": "",
                 "untags": [],
-                "value": "org.apache.http.entity.ByteArrayEntity.(byte[],int,int,org.apache.http.entity.ContentType)"
+                "value": "java.io.ByteArrayOutputStream.write(byte[],int,int)"
             },
             {
                 "command": "",
@@ -4796,12 +4658,12 @@
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "O",
-                "track": "false",
+                "track": "",
                 "untags": [],
-                "value": "org.apache.http.entity.ByteArrayEntity.(byte[],org.apache.http.entity.ContentType)"
+                "value": "java.io.CharArrayReader.(char[])"
             },
             {
-                "command": "",
+                "command": "INSERT(0,P2,P3)",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
@@ -4809,22 +4671,22 @@
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "O",
-                "track": "false",
+                "track": "",
                 "untags": [],
-                "value": "org.apache.http.entity.InputStreamEntity.(java.io.InputStream,long,org.apache.http.entity.ContentType)"
+                "value": "java.io.CharArrayReader.(char[],int,int)"
             },
             {
-                "command": "",
+                "command": "INSERT(0,P2,P3)",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
-                "source": "P1",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "O",
-                "track": "false",
+                "target": "P1",
+                "track": "",
                 "untags": [],
-                "value": "org.apache.http.entity.StringEntity.(java.lang.String,java.lang.String,java.lang.String)"
+                "value": "java.io.CharArrayReader.read(char[],int,int)"
             },
             {
                 "command": "",
@@ -4835,17 +4697,10 @@
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "O",
-                "track": "false",
+                "track": "",
                 "untags": [],
-                "value": "org.apache.http.entity.StringEntity.(java.lang.String,org.apache.http.entity.ContentType)"
-            }
-        ],
-        "enable": 1,
-        "type": 1,
-        "value": "httpclient"
-    },
-    {
-        "details": [
+                "value": "java.io.FileReader.(java.io.File)"
+            },
             {
                 "command": "",
                 "ignore_blacklist": false,
@@ -4855,22 +4710,22 @@
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "O",
-                "track": "false",
+                "track": "",
                 "untags": [],
-                "value": "org.apache.hc.core5.http.io.entity.ByteArrayEntity.(byte[],int,int,org.apache.hc.core5.http.ContentType,java.lang.String,boolean)"
+                "value": "java.io.InputStream.(java.io.InputStream)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
-                "source": "P1",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "O",
-                "track": "false",
+                "target": "P1",
+                "track": "",
                 "untags": [],
-                "value": "org.apache.hc.core5.http.io.entity.ByteArrayEntity.(byte[],org.apache.hc.core5.http.ContentType,java.lang.String,boolean)"
+                "value": "java.io.InputStream.read(byte[],int,int)"
             },
             {
                 "command": "",
@@ -4881,9 +4736,9 @@
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "O",
-                "track": "false",
+                "track": "",
                 "untags": [],
-                "value": "org.apache.hc.core5.http.io.entity.InputStreamEntity.(java.io.InputStream,long,org.apache.hc.core5.http.ContentType,java.lang.String)"
+                "value": "java.io.InputStreamReader.(java.io.InputStream)"
             },
             {
                 "command": "",
@@ -4894,45 +4749,38 @@
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "O",
-                "track": "false",
+                "track": "",
                 "untags": [],
-                "value": "org.apache.hc.core5.http.io.entity.StringEntity.(java.lang.String,org.apache.hc.core5.http.ContentType,java.lang.String,boolean)"
-            }
-        ],
-        "enable": 1,
-        "type": 1,
-        "value": "httpclient5"
-    },
-    {
-        "details": [
+                "value": "java.io.InputStreamReader.(java.io.InputStream,java.nio.charset.Charset)"
+            },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
-                "source": "P1",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "O",
+                "target": "P1",
                 "track": "",
                 "untags": [],
-                "value": "java.io.BufferedReader.(java.io.InputStreamReader)"
+                "value": "java.io.InputStreamReader.read(char[],int,int)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "all",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "O",
                 "track": "",
                 "untags": [],
-                "value": "java.io.BufferedReader.(java.io.Reader)"
+                "value": "java.io.ObjectInputStream.(java.io.InputStream)"
             },
             {
-                "command": "",
+                "command": "INSERT(0,P2,P3)",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
@@ -4940,22 +4788,22 @@
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "O",
-                "track": "",
+                "track": "false",
                 "untags": [],
-                "value": "java.io.BufferedReader.(java.io.Reader,int)"
+                "value": "java.io.PipedInputStream.read(byte[],int,int)"
             },
             {
-                "command": "",
+                "command": "INSERT(0,P2,P3)",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
-                "source": "O",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "R",
-                "track": "",
+                "target": "O",
+                "track": "false",
                 "untags": [],
-                "value": "java.io.BufferedReader.readLine()"
+                "value": "java.io.PipedReader.read(char[],int,int)"
             },
             {
                 "command": "",
@@ -4968,33 +4816,33 @@
                 "target": "O",
                 "track": "",
                 "untags": [],
-                "value": "java.io.ByteArrayInputStream.(byte[])"
+                "value": "java.io.PushbackInputStream.(java.io.InputStream,int)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
-                "source": "P1",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "O",
-                "track": "",
+                "target": "P1",
+                "track": "false",
                 "untags": [],
-                "value": "java.io.ByteArrayInputStream.(byte[],int,int)"
+                "value": "java.io.PushbackInputStream.read(byte[],int,int)"
             },
             {
-                "command": "REMOVE()",
+                "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
-                "source": "O",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "O",
                 "track": "",
                 "untags": [],
-                "value": "java.io.ByteArrayOutputStream.reset()"
+                "value": "java.io.StringReader.(java.lang.String)"
             },
             {
                 "command": "KEEP()",
@@ -5007,88 +4855,88 @@
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "java.io.ByteArrayOutputStream.toByteArray()"
+                "value": "java.io.StringWriter.toString()"
             },
             {
-                "command": "KEEP()",
+                "command": "APPEND(P2,P3)",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
-                "source": "O",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "R",
+                "target": "O",
                 "track": "false",
                 "untags": [],
-                "value": "java.io.ByteArrayOutputStream.toString()"
+                "value": "java.io.StringWriter.write(char[],int,int)"
             },
             {
-                "command": "KEEP()",
+                "command": "APPEND()",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
-                "source": "O",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "R",
+                "target": "O",
                 "track": "false",
                 "untags": [],
-                "value": "java.io.ByteArrayOutputStream.toString(int)"
+                "value": "java.io.StringWriter.write(java.lang.String)"
             },
             {
-                "command": "KEEP()",
+                "command": "APPEND(P2,P3)",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
-                "source": "O",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "R",
+                "target": "O",
                 "track": "false",
                 "untags": [],
-                "value": "java.io.ByteArrayOutputStream.toString(java.lang.String)"
+                "value": "java.io.StringWriter.write(java.lang.String,int,int)"
             },
             {
-                "command": "KEEP()",
+                "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
-                "source": "O",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "R",
-                "track": "false",
+                "target": "O",
+                "track": "",
                 "untags": [],
-                "value": "java.io.ByteArrayOutputStream.toString(java.nio.charset.Charset)"
+                "value": "java.net.Socket.(java.lang.String,int)"
             },
             {
-                "command": "APPEND(P2,P3)",
+                "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
-                "source": "P1",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "O",
+                "target": "R",
                 "track": "",
                 "untags": [],
-                "value": "java.io.ByteArrayOutputStream.write(byte[],int,int)"
+                "value": "java.net.Socket.getOutputStream()"
             },
             {
-                "command": "",
+                "command": "REMOVE()",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
-                "source": "P1",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "O",
-                "track": "",
+                "track": "false",
                 "untags": [],
-                "value": "java.io.CharArrayReader.(char[])"
+                "value": "org.apache.commons.io.output.ByteArrayOutputStream.reset()"
             },
             {
-                "command": "INSERT(0,P2,P3)",
+                "command": "APPEND(P2,P3)",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
@@ -5096,116 +4944,123 @@
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "O",
-                "track": "",
+                "track": "false",
                 "untags": [],
-                "value": "java.io.CharArrayReader.(char[],int,int)"
-            },
+                "value": "org.apache.commons.io.output.ByteArrayOutputStream.write(byte[],int,int)"
+            }
+        ],
+        "enable": 1,
+        "type": 1,
+        "value": "io"
+    },
+    {
+        "details": [
             {
-                "command": "INSERT(0,P2,P3)",
+                "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
                 "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "P1",
+                "target": "R",
                 "track": "",
                 "untags": [],
-                "value": "java.io.CharArrayReader.read(char[],int,int)"
+                "value": "javax.xml.bind.JAXBElement.getValue()"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "O",
+                "target": "R",
                 "track": "",
                 "untags": [],
-                "value": "java.io.FileReader.(java.io.File)"
+                "value": "javax.xml.stream.XMLInputFactory.createXMLStreamReader(java.io.InputStream)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "O",
+                "target": "R",
                 "track": "",
                 "untags": [],
-                "value": "java.io.InputStream.(java.io.InputStream)"
+                "value": "javax.xml.stream.XMLInputFactory.createXMLStreamReader(java.io.InputStream,java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "O",
+                "inherit": "true",
+                "source": "P2",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "P1",
+                "target": "R",
                 "track": "",
                 "untags": [],
-                "value": "java.io.InputStream.read(byte[],int,int)"
+                "value": "javax.xml.stream.XMLInputFactory.createXMLStreamReader(java.lang.String,java.io.InputStream)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
-                "stack_blacklist": [],
+                "inherit": "true",
+                "source": "P2",
+                "stack_blacklist": [],
                 "tags": [],
-                "target": "O",
+                "target": "R",
                 "track": "",
                 "untags": [],
-                "value": "java.io.InputStreamReader.(java.io.InputStream)"
+                "value": "javax.xml.stream.XMLInputFactory.createXMLStreamReader(java.lang.String,java.io.Reader)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "O",
+                "target": "R",
                 "track": "",
                 "untags": [],
-                "value": "java.io.InputStreamReader.(java.io.InputStream,java.nio.charset.Charset)"
+                "value": "javax.xml.stream.XMLInputFactory.createXMLStreamReader(javax.xml.transform.Source)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
-                "source": "O",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "P1",
+                "target": "O",
                 "track": "",
                 "untags": [],
-                "value": "java.io.InputStreamReader.read(char[],int,int)"
+                "value": "javax.xml.transform.sax.SAXSource.(org.xml.sax.InputSource)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
+                "inherit": "false",
+                "source": "P2",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "O",
                 "track": "",
                 "untags": [],
-                "value": "java.io.ObjectInputStream.(java.io.InputStream)"
+                "value": "javax.xml.transform.sax.SAXSource.(org.xml.sax.XMLReader,org.xml.sax.InputSource)"
             },
             {
-                "command": "INSERT(0,P2,P3)",
+                "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
@@ -5213,12 +5068,12 @@
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "O",
-                "track": "false",
+                "track": "",
                 "untags": [],
-                "value": "java.io.PipedInputStream.read(byte[],int,int)"
+                "value": "javax.xml.transform.sax.SAXSource.setInputSource(org.xml.sax.InputSource)"
             },
             {
-                "command": "INSERT(0,P2,P3)",
+                "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
@@ -5226,9 +5081,9 @@
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "O",
-                "track": "false",
+                "track": "",
                 "untags": [],
-                "value": "java.io.PipedReader.read(char[],int,int)"
+                "value": "javax.xml.transform.stream.StreamSource.(java.io.File)"
             },
             {
                 "command": "",
@@ -5241,20 +5096,20 @@
                 "target": "O",
                 "track": "",
                 "untags": [],
-                "value": "java.io.PushbackInputStream.(java.io.InputStream,int)"
+                "value": "javax.xml.transform.stream.StreamSource.(java.io.InputStream)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
-                "source": "O",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "P1",
-                "track": "false",
+                "target": "O",
+                "track": "",
                 "untags": [],
-                "value": "java.io.PushbackInputStream.read(byte[],int,int)"
+                "value": "javax.xml.transform.stream.StreamSource.(java.io.InputStream,java.lang.String)"
             },
             {
                 "command": "",
@@ -5267,144 +5122,152 @@
                 "target": "O",
                 "track": "",
                 "untags": [],
-                "value": "java.io.StringReader.(java.lang.String)"
+                "value": "javax.xml.transform.stream.StreamSource.(java.io.Reader)"
             },
             {
-                "command": "KEEP()",
+                "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
-                "source": "O",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "R",
-                "track": "false",
+                "target": "O",
+                "track": "",
                 "untags": [],
-                "value": "java.io.StringWriter.toString()"
-            },
+                "value": "javax.xml.transform.stream.StreamSource.(java.io.Reader,java.lang.String)"
+            }
+        ],
+        "enable": 1,
+        "type": 1,
+        "value": "javax"
+    },
+    {
+        "details": [
             {
-                "command": "APPEND(P2,P3)",
+                "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
+                "inherit": "true",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "O",
+                "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "java.io.StringWriter.write(char[],int,int)"
+                "value": "jakarta.servlet.ServletRequest.getInputStream()"
             },
             {
-                "command": "APPEND()",
+                "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "O",
+                "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "java.io.StringWriter.write(java.lang.String)"
+                "value": "jakarta.servlet.ServletRequest.getParameter(java.lang.String)"
             },
             {
-                "command": "APPEND(P2,P3)",
+                "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
+                "inherit": "true",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "O",
+                "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "java.io.StringWriter.write(java.lang.String,int,int)"
+                "value": "jakarta.servlet.ServletRequest.getParameterNames()"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "O",
-                "track": "",
+                "target": "R",
+                "track": "false",
                 "untags": [],
-                "value": "java.net.Socket.(java.lang.String,int)"
+                "value": "jakarta.servlet.ServletRequest.getParameterValues(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "true",
                 "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "R",
-                "track": "",
+                "track": "false",
                 "untags": [],
-                "value": "java.net.Socket.getOutputStream()"
+                "value": "jakarta.servlet.ServletRequest.getReader()"
             },
             {
-                "command": "REMOVE()",
+                "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "true",
                 "source": "O",
                 "stack_blacklist": [],
-                "tags": [],
-                "target": "O",
+                "tags": [
+                    "cross-site"
+                ],
+                "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "org.apache.commons.io.output.ByteArrayOutputStream.reset()"
+                "value": "javax.servlet.ServletRequest.getInputStream()"
             },
             {
-                "command": "APPEND(P2,P3)",
+                "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
-                "tags": [],
-                "target": "O",
+                "tags": [
+                    "cross-site"
+                ],
+                "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "org.apache.commons.io.output.ByteArrayOutputStream.write(byte[],int,int)"
-            }
-        ],
-        "enable": 1,
-        "type": 1,
-        "value": "io"
-    },
-    {
-        "details": [
+                "value": "javax.servlet.ServletRequest.getParameter(java.lang.String)"
+            },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "true",
                 "source": "O",
                 "stack_blacklist": [],
-                "tags": [],
+                "tags": [
+                    "cross-site"
+                ],
                 "target": "R",
-                "track": "",
+                "track": "false",
                 "untags": [],
-                "value": "javax.xml.bind.JAXBElement.getValue()"
+                "value": "javax.servlet.ServletRequest.getParameterMap()"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "P1",
+                "source": "O",
                 "stack_blacklist": [],
-                "tags": [],
+                "tags": [
+                    "http-token-limited-chars"
+                ],
                 "target": "R",
-                "track": "",
+                "track": "false",
                 "untags": [],
-                "value": "javax.xml.stream.XMLInputFactory.createXMLStreamReader(java.io.InputStream)"
+                "value": "javax.servlet.ServletRequest.getParameterNames()"
             },
             {
                 "command": "",
@@ -5413,162 +5276,186 @@
                 "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
-                "tags": [],
+                "tags": [
+                    "cross-site"
+                ],
                 "target": "R",
-                "track": "",
+                "track": "false",
                 "untags": [],
-                "value": "javax.xml.stream.XMLInputFactory.createXMLStreamReader(java.io.InputStream,java.lang.String)"
+                "value": "javax.servlet.ServletRequest.getParameterValues(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "P2",
+                "source": "O",
                 "stack_blacklist": [],
-                "tags": [],
+                "tags": [
+                    "cross-site"
+                ],
                 "target": "R",
-                "track": "",
+                "track": "false",
                 "untags": [],
-                "value": "javax.xml.stream.XMLInputFactory.createXMLStreamReader(java.lang.String,java.io.InputStream)"
-            },
+                "value": "javax.servlet.ServletRequest.getReader()"
+            }
+        ],
+        "enable": 1,
+        "type": 2,
+        "value": "javax.servlet.ServletRequest"
+    },
+    {
+        "details": [
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "P2",
+                "source": "O",
                 "stack_blacklist": [],
-                "tags": [],
+                "tags": [
+                    "cross-site"
+                ],
                 "target": "R",
-                "track": "",
+                "track": "false",
                 "untags": [],
-                "value": "javax.xml.stream.XMLInputFactory.createXMLStreamReader(java.lang.String,java.io.Reader)"
+                "value": "jakarta.servlet.ServletRequest.getParameterMap()"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "P1",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "R",
-                "track": "",
+                "track": "false",
                 "untags": [],
-                "value": "javax.xml.stream.XMLInputFactory.createXMLStreamReader(javax.xml.transform.Source)"
+                "value": "jakarta.servlet.http.HttpServletRequest.getCookies()"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
-                "tags": [],
-                "target": "O",
-                "track": "",
+                "tags": [
+                    "cross-site"
+                ],
+                "target": "R",
+                "track": "false",
                 "untags": [],
-                "value": "javax.xml.transform.sax.SAXSource.(org.xml.sax.InputSource)"
+                "value": "jakarta.servlet.http.HttpServletRequest.getHeader(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "P2",
+                "inherit": "true",
+                "source": "O",
                 "stack_blacklist": [],
-                "tags": [],
-                "target": "O",
-                "track": "",
+                "tags": [
+                    "http-token-limited-chars"
+                ],
+                "target": "R",
+                "track": "false",
                 "untags": [],
-                "value": "javax.xml.transform.sax.SAXSource.(org.xml.sax.XMLReader,org.xml.sax.InputSource)"
+                "value": "jakarta.servlet.http.HttpServletRequest.getHeaderNames()"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
-                "tags": [],
-                "target": "O",
-                "track": "",
+                "tags": [
+                    "cross-site"
+                ],
+                "target": "R",
+                "track": "false",
                 "untags": [],
-                "value": "javax.xml.transform.sax.SAXSource.setInputSource(org.xml.sax.InputSource)"
+                "value": "jakarta.servlet.http.HttpServletRequest.getHeaders(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
-                "tags": [],
-                "target": "O",
-                "track": "",
+                "tags": [
+                    "cross-site"
+                ],
+                "target": "R",
+                "track": "false",
                 "untags": [],
-                "value": "javax.xml.transform.stream.StreamSource.(java.io.File)"
+                "value": "jakarta.servlet.http.HttpServletRequest.getParameter(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
-                "tags": [],
-                "target": "O",
-                "track": "",
+                "tags": [
+                    "cross-site"
+                ],
+                "target": "R",
+                "track": "false",
                 "untags": [],
-                "value": "javax.xml.transform.stream.StreamSource.(java.io.InputStream)"
+                "value": "jakarta.servlet.http.HttpServletRequest.getPart(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
+                "inherit": "true",
+                "source": "O",
                 "stack_blacklist": [],
-                "tags": [],
-                "target": "O",
-                "track": "",
+                "tags": [
+                    "cross-site"
+                ],
+                "target": "R",
+                "track": "false",
                 "untags": [],
-                "value": "javax.xml.transform.stream.StreamSource.(java.io.InputStream,java.lang.String)"
+                "value": "jakarta.servlet.http.HttpServletRequest.getParts()"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
+                "inherit": "true",
+                "source": "O",
                 "stack_blacklist": [],
-                "tags": [],
-                "target": "O",
-                "track": "",
+                "tags": [
+                    "cross-site",
+                    "xss-encoded"
+                ],
+                "target": "R",
+                "track": "false",
                 "untags": [],
-                "value": "javax.xml.transform.stream.StreamSource.(java.io.Reader)"
+                "value": "jakarta.servlet.http.HttpServletRequest.getQueryString()"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
+                "inherit": "true",
+                "source": "O",
                 "stack_blacklist": [],
-                "tags": [],
-                "target": "O",
-                "track": "",
+                "tags": [
+                    "http-token-limited-chars",
+                    "xss-encoded"
+                ],
+                "target": "R",
+                "track": "false",
                 "untags": [],
-                "value": "javax.xml.transform.stream.StreamSource.(java.io.Reader,java.lang.String)"
-            }
-        ],
-        "enable": 1,
-        "type": 1,
-        "value": "javax"
-    },
-    {
-        "details": [
+                "value": "jakarta.servlet.http.HttpServletRequest.getRequestedSessionId()"
+            },
             {
                 "command": "",
                 "ignore_blacklist": false,
@@ -5580,7 +5467,7 @@
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "jakarta.servlet.ServletRequest.getInputStream()"
+                "value": "jakarta.servlet.http.Part.getContentType()"
             },
             {
                 "command": "",
@@ -5593,7 +5480,7 @@
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "jakarta.servlet.ServletRequest.getParameter(java.lang.String)"
+                "value": "jakarta.servlet.http.Part.getHeader(java.lang.String)"
             },
             {
                 "command": "",
@@ -5602,11 +5489,13 @@
                 "inherit": "true",
                 "source": "O",
                 "stack_blacklist": [],
-                "tags": [],
+                "tags": [
+                    "http-token-limited-chars"
+                ],
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "jakarta.servlet.ServletRequest.getParameterNames()"
+                "value": "jakarta.servlet.http.Part.getHeaderNames()"
             },
             {
                 "command": "",
@@ -5619,7 +5508,7 @@
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "jakarta.servlet.ServletRequest.getParameterValues(java.lang.String)"
+                "value": "jakarta.servlet.http.Part.getHeaders(java.lang.String)"
             },
             {
                 "command": "",
@@ -5628,11 +5517,13 @@
                 "inherit": "true",
                 "source": "O",
                 "stack_blacklist": [],
-                "tags": [],
+                "tags": [
+                    "cross-site"
+                ],
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "jakarta.servlet.ServletRequest.getReader()"
+                "value": "jakarta.servlet.http.Part.getInputStream()"
             },
             {
                 "command": "",
@@ -5641,28 +5532,24 @@
                 "inherit": "true",
                 "source": "O",
                 "stack_blacklist": [],
-                "tags": [
-                    "cross-site"
-                ],
+                "tags": [],
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "javax.servlet.ServletRequest.getInputStream()"
+                "value": "jakarta.servlet.http.Part.getName()"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "P1",
+                "source": "O",
                 "stack_blacklist": [],
-                "tags": [
-                    "cross-site"
-                ],
+                "tags": [],
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "javax.servlet.ServletRequest.getParameter(java.lang.String)"
+                "value": "jakarta.servlet.http.Part.getSubmittedFileName()"
             },
             {
                 "command": "",
@@ -5671,50 +5558,46 @@
                 "inherit": "true",
                 "source": "O",
                 "stack_blacklist": [],
-                "tags": [
-                    "cross-site"
-                ],
+                "tags": [],
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "javax.servlet.ServletRequest.getParameterMap()"
+                "value": "javax.servlet.http.HttpServletRequest.getCookies()"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "O",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [
-                    "http-token-limited-chars"
+                    "cross-site"
                 ],
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "javax.servlet.ServletRequest.getParameterNames()"
+                "value": "javax.servlet.http.HttpServletRequest.getHeader(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "P1",
+                "source": "O",
                 "stack_blacklist": [],
-                "tags": [
-                    "cross-site"
-                ],
+                "tags": [],
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "javax.servlet.ServletRequest.getParameterValues(java.lang.String)"
+                "value": "javax.servlet.http.HttpServletRequest.getHeaderNames()"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "O",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [
                     "cross-site"
@@ -5722,15 +5605,8 @@
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "javax.servlet.ServletRequest.getReader()"
-            }
-        ],
-        "enable": 1,
-        "type": 2,
-        "value": "javax.servlet.ServletRequest"
-    },
-    {
-        "details": [
+                "value": "javax.servlet.http.HttpServletRequest.getHeaders(java.lang.String)"
+            },
             {
                 "command": "",
                 "ignore_blacklist": false,
@@ -5744,27 +5620,29 @@
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "jakarta.servlet.ServletRequest.getParameterMap()"
+                "value": "javax.servlet.http.HttpServletRequest.getInputStream()"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "O",
+                "source": "P1",
                 "stack_blacklist": [],
-                "tags": [],
+                "tags": [
+                    "cross-site"
+                ],
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "jakarta.servlet.http.HttpServletRequest.getCookies()"
+                "value": "javax.servlet.http.HttpServletRequest.getParameter(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "P1",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [
                     "cross-site"
@@ -5772,7 +5650,7 @@
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "jakarta.servlet.http.HttpServletRequest.getHeader(java.lang.String)"
+                "value": "javax.servlet.http.HttpServletRequest.getParameterMap()"
             },
             {
                 "command": "",
@@ -5782,12 +5660,12 @@
                 "source": "O",
                 "stack_blacklist": [],
                 "tags": [
-                    "http-token-limited-chars"
+                    "cross-site"
                 ],
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "jakarta.servlet.http.HttpServletRequest.getHeaderNames()"
+                "value": "javax.servlet.http.HttpServletRequest.getParameterNames()"
             },
             {
                 "command": "",
@@ -5802,7 +5680,7 @@
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "jakarta.servlet.http.HttpServletRequest.getHeaders(java.lang.String)"
+                "value": "javax.servlet.http.HttpServletRequest.getParameterValues(java.lang.String)"
             },
             {
                 "command": "",
@@ -5817,14 +5695,14 @@
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "jakarta.servlet.http.HttpServletRequest.getParameter(java.lang.String)"
+                "value": "javax.servlet.http.HttpServletRequest.getPart(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "P1",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [
                     "cross-site"
@@ -5832,7 +5710,7 @@
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "jakarta.servlet.http.HttpServletRequest.getPart(java.lang.String)"
+                "value": "javax.servlet.http.HttpServletRequest.getParts()"
             },
             {
                 "command": "",
@@ -5842,12 +5720,13 @@
                 "source": "O",
                 "stack_blacklist": [],
                 "tags": [
-                    "cross-site"
+                    "cross-site",
+                    "xss-encoded"
                 ],
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "jakarta.servlet.http.HttpServletRequest.getParts()"
+                "value": "javax.servlet.http.HttpServletRequest.getQueryString()"
             },
             {
                 "command": "",
@@ -5857,13 +5736,12 @@
                 "source": "O",
                 "stack_blacklist": [],
                 "tags": [
-                    "cross-site",
-                    "xss-encoded"
+                    "cross-site"
                 ],
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "jakarta.servlet.http.HttpServletRequest.getQueryString()"
+                "value": "javax.servlet.http.HttpServletRequest.getReader()"
             },
             {
                 "command": "",
@@ -5879,7 +5757,7 @@
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "jakarta.servlet.http.HttpServletRequest.getRequestedSessionId()"
+                "value": "javax.servlet.http.HttpServletRequest.getRequestedSessionId()"
             },
             {
                 "command": "",
@@ -5892,7 +5770,7 @@
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "jakarta.servlet.http.Part.getContentType()"
+                "value": "javax.servlet.http.Part.getContentType()"
             },
             {
                 "command": "",
@@ -5905,7 +5783,7 @@
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "jakarta.servlet.http.Part.getHeader(java.lang.String)"
+                "value": "javax.servlet.http.Part.getHeader(java.lang.String)"
             },
             {
                 "command": "",
@@ -5920,7 +5798,7 @@
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "jakarta.servlet.http.Part.getHeaderNames()"
+                "value": "javax.servlet.http.Part.getHeaderNames()"
             },
             {
                 "command": "",
@@ -5933,7 +5811,7 @@
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "jakarta.servlet.http.Part.getHeaders(java.lang.String)"
+                "value": "javax.servlet.http.Part.getHeaders(java.lang.String)"
             },
             {
                 "command": "",
@@ -5948,7 +5826,7 @@
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "jakarta.servlet.http.Part.getInputStream()"
+                "value": "javax.servlet.http.Part.getInputStream()"
             },
             {
                 "command": "",
@@ -5961,7 +5839,7 @@
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "jakarta.servlet.http.Part.getName()"
+                "value": "javax.servlet.http.Part.getName()"
             },
             {
                 "command": "",
@@ -5974,124 +5852,133 @@
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "jakarta.servlet.http.Part.getSubmittedFileName()"
-            },
+                "value": "javax.servlet.http.Part.getSubmittedFileName()"
+            }
+        ],
+        "enable": 1,
+        "type": 2,
+        "value": "javax.servlet.http.HttpServletRequest"
+    },
+    {
+        "details": [
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "O",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "R",
-                "track": "false",
+                "target": "O",
+                "track": "",
                 "untags": [],
-                "value": "javax.servlet.http.HttpServletRequest.getCookies()"
-            },
+                "value": "java.sql.Connection.nativeSQL(java.lang.String)"
+            }
+        ],
+        "enable": 1,
+        "type": 1,
+        "value": "jdbc"
+    },
+    {
+        "details": [
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "P1",
+                "source": "P2",
                 "stack_blacklist": [],
-                "tags": [
-                    "cross-site"
-                ],
-                "target": "R",
-                "track": "false",
+                "tags": [],
+                "target": "",
+                "track": "true",
                 "untags": [],
-                "value": "javax.servlet.http.HttpServletRequest.getHeader(java.lang.String)"
+                "value": "jakarta.naming.directory.DirContext.search(java.lang.String,java.lang.String,jakarta.naming.directory.SearchControls)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "O",
+                "source": "P2",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "R",
-                "track": "false",
+                "target": "",
+                "track": "true",
                 "untags": [],
-                "value": "javax.servlet.http.HttpServletRequest.getHeaderNames()"
+                "value": "jakarta.naming.directory.DirContext.search(java.lang.String,java.lang.String,java.lang.Object[],jakarta.naming.directory.SearchControls)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "P1",
+                "source": "P2",
                 "stack_blacklist": [],
-                "tags": [
-                    "cross-site"
-                ],
-                "target": "R",
-                "track": "false",
+                "tags": [],
+                "target": "",
+                "track": "true",
                 "untags": [],
-                "value": "javax.servlet.http.HttpServletRequest.getHeaders(java.lang.String)"
+                "value": "jakarta.naming.directory.InitialDirContext.search(java.lang.String,java.lang.String,jakarta.naming.directory.SearchControls)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
-                "source": "O",
+                "inherit": "all",
+                "source": "P2",
                 "stack_blacklist": [],
-                "tags": [
-                    "cross-site"
-                ],
-                "target": "R",
-                "track": "false",
+                "tags": [],
+                "target": "",
+                "track": "true",
                 "untags": [],
-                "value": "javax.servlet.http.HttpServletRequest.getInputStream()"
+                "value": "javax.naming.directory.DirContext.search(java.lang.String,java.lang.String,java.lang.Object[],javax.naming.directory.SearchControls)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
-                "source": "P1",
+                "inherit": "all",
+                "source": "P2",
                 "stack_blacklist": [],
-                "tags": [
-                    "cross-site"
-                ],
-                "target": "R",
-                "track": "false",
+                "tags": [],
+                "target": "",
+                "track": "true",
                 "untags": [],
-                "value": "javax.servlet.http.HttpServletRequest.getParameter(java.lang.String)"
+                "value": "javax.naming.directory.DirContext.search(java.lang.String,java.lang.String,javax.naming.directory.SearchControls)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
-                "source": "O",
+                "inherit": "all",
+                "source": "P2",
                 "stack_blacklist": [],
-                "tags": [
-                    "cross-site"
-                ],
-                "target": "R",
-                "track": "false",
+                "tags": [],
+                "target": "",
+                "track": "true",
                 "untags": [],
-                "value": "javax.servlet.http.HttpServletRequest.getParameterMap()"
+                "value": "javax.naming.directory.InitialDirContext.search(java.lang.String,java.lang.String,java.lang.Object[],javax.naming.directory.SearchControls)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
-                "source": "O",
+                "inherit": "all",
+                "source": "P2",
                 "stack_blacklist": [],
-                "tags": [
-                    "cross-site"
-                ],
-                "target": "R",
-                "track": "false",
+                "tags": [],
+                "target": "",
+                "track": "true",
                 "untags": [],
-                "value": "javax.servlet.http.HttpServletRequest.getParameterNames()"
-            },
+                "value": "javax.naming.directory.InitialDirContext.search(java.lang.String,java.lang.String,javax.naming.directory.SearchControls)"
+            }
+        ],
+        "enable": 1,
+        "type": 4,
+        "value": "ldap-injection"
+    },
+    {
+        "details": [
             {
                 "command": "",
                 "ignore_blacklist": false,
@@ -6099,14 +5986,19 @@
                 "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
-                "tags": [
-                    "cross-site"
-                ],
-                "target": "R",
-                "track": "false",
+                "tags": [],
+                "target": "",
+                "track": "true",
                 "untags": [],
-                "value": "javax.servlet.http.HttpServletRequest.getParameterValues(java.lang.String)"
-            },
+                "value": "com.mongodb.DB.doEval(java.lang.String,java.lang.Object[])"
+            }
+        ],
+        "enable": 1,
+        "type": 4,
+        "value": "nosql-injection"
+    },
+    {
+        "details": [
             {
                 "command": "",
                 "ignore_blacklist": false,
@@ -6114,175 +6006,173 @@
                 "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
-                "tags": [
-                    "cross-site"
-                ],
+                "tags": [],
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "javax.servlet.http.HttpServletRequest.getPart(java.lang.String)"
+                "value": "com.opensymphony.xwork2.util.ValueStack.findString(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "O",
+                "source": "P1",
                 "stack_blacklist": [],
-                "tags": [
-                    "cross-site"
-                ],
+                "tags": [],
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "javax.servlet.http.HttpServletRequest.getParts()"
+                "value": "com.opensymphony.xwork2.util.ValueStack.findValue(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "O",
+                "source": "P1",
                 "stack_blacklist": [],
-                "tags": [
-                    "cross-site",
-                    "xss-encoded"
-                ],
+                "tags": [],
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "javax.servlet.http.HttpServletRequest.getQueryString()"
+                "value": "com.opensymphony.xwork2.util.ValueStack.findValue(java.lang.String,java.lang.Class)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
-                "source": "O",
+                "inherit": "false",
+                "source": "P1",
                 "stack_blacklist": [],
-                "tags": [
-                    "cross-site"
-                ],
+                "tags": [],
                 "target": "R",
-                "track": "false",
+                "track": "",
                 "untags": [],
-                "value": "javax.servlet.http.HttpServletRequest.getReader()"
+                "value": "ognl.Ognl.parseExpression(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
-                "source": "O",
+                "inherit": "false",
+                "source": "P1",
                 "stack_blacklist": [],
-                "tags": [
-                    "http-token-limited-chars",
-                    "xss-encoded"
-                ],
-                "target": "R",
-                "track": "false",
+                "tags": [],
+                "target": "O",
+                "track": "",
                 "untags": [],
-                "value": "javax.servlet.http.HttpServletRequest.getRequestedSessionId()"
+                "value": "ognl.OgnlParser.(java.io.InputStream)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
-                "source": "O",
+                "inherit": "false",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "R",
-                "track": "false",
+                "target": "O",
+                "track": "",
                 "untags": [],
-                "value": "javax.servlet.http.Part.getContentType()"
+                "value": "ognl.OgnlParser.(java.io.Reader)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
+                "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "R",
-                "track": "false",
+                "target": "O",
+                "track": "",
                 "untags": [],
-                "value": "javax.servlet.http.Part.getHeader(java.lang.String)"
+                "value": "ognl.OgnlParser.(ognl.OgnlParserTokenManager)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
+                "inherit": "false",
                 "source": "O",
                 "stack_blacklist": [],
-                "tags": [
-                    "http-token-limited-chars"
-                ],
+                "tags": [],
                 "target": "R",
-                "track": "false",
+                "track": "",
                 "untags": [],
-                "value": "javax.servlet.http.Part.getHeaderNames()"
-            },
+                "value": "ognl.OgnlParser.topLevelExpression()"
+            }
+        ],
+        "enable": 1,
+        "type": 1,
+        "value": "ognl"
+    },
+    {
+        "details": [
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
-                "source": "P1",
+                "inherit": "false",
+                "source": "P2",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "javax.servlet.http.Part.getHeaders(java.lang.String)"
-            },
+                "value": "com.squareup.okhttp.RequestBody.create(com.squareup.okhttp.MediaType,byte[],int,int)"
+            }
+        ],
+        "enable": 1,
+        "type": 1,
+        "value": "okhttp"
+    },
+    {
+        "details": [
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
-                "source": "O",
+                "inherit": "false",
+                "source": "P1",
                 "stack_blacklist": [],
-                "tags": [
-                    "cross-site"
-                ],
+                "tags": [],
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "javax.servlet.http.Part.getInputStream()"
+                "value": "okhttp3.RequestBody$Companion.create(byte[],okhttp3.MediaType,int,int)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
-                "source": "O",
+                "inherit": "false",
+                "source": "P2",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "javax.servlet.http.Part.getName()"
+                "value": "okhttp3.RequestBody$Companion.create(okhttp3.MediaType,byte[],int,int)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
-                "source": "O",
+                "inherit": "false",
+                "source": "P2",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "R",
                 "track": "false",
                 "untags": [],
-                "value": "javax.servlet.http.Part.getSubmittedFileName()"
+                "value": "okhttp3.RequestBody.create(okhttp3.MediaType,byte[],int,int)"
             }
         ],
         "enable": 1,
-        "type": 2,
-        "value": "javax.servlet.http.HttpServletRequest"
+        "type": 1,
+        "value": "okhttp3"
     },
     {
         "details": [
@@ -6291,18 +6181,18 @@
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "P1",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "O",
-                "track": "",
+                "target": "R",
+                "track": "false",
                 "untags": [],
-                "value": "java.sql.Connection.nativeSQL(java.lang.String)"
+                "value": "org.apache.commons.fileupload.FileItem.getString()"
             }
         ],
         "enable": 1,
         "type": 1,
-        "value": "jdbc"
+        "value": "org.apache.commons.fileupload.FileItem"
     },
     {
         "details": [
@@ -6310,117 +6200,71 @@
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
-                "source": "P2",
+                "inherit": "all",
+                "source": "P1",
                 "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
+                "tags": [
+                    "cross-site"
+                ],
+                "target": "R",
+                "track": "false",
                 "untags": [],
-                "value": "jakarta.naming.directory.DirContext.search(java.lang.String,java.lang.String,jakarta.naming.directory.SearchControls)"
-            },
+                "value": "org.apache.commons.fileupload.FileUploadBase.parseRequest(org.apache.commons.fileupload.RequestContext)"
+            }
+        ],
+        "enable": 1,
+        "type": 2,
+        "value": "org.apache.commons.fileupload.FileUploadBase"
+    },
+    {
+        "details": [
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "P2",
+                "source": "P1",
                 "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
+                "tags": [
+                    "cross-site"
+                ],
+                "target": "R",
+                "track": "false",
                 "untags": [],
-                "value": "jakarta.naming.directory.DirContext.search(java.lang.String,java.lang.String,java.lang.Object[],jakarta.naming.directory.SearchControls)"
+                "value": "org.springframework.web.bind.annotation.support.HandlerMethodInvoker.resolvePathVariable(java.lang.String,org.springframework.core.MethodParameter,org.springframework.web.context.request.NativeWebRequest,java.lang.Object)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "P2",
+                "source": "P1",
                 "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
+                "tags": [
+                    "cross-site"
+                ],
+                "target": "R",
+                "track": "false",
                 "untags": [],
-                "value": "jakarta.naming.directory.InitialDirContext.search(java.lang.String,java.lang.String,jakarta.naming.directory.SearchControls)"
+                "value": "org.springframework.web.servlet.mvc.method.annotation.PathVariableMethodArgumentResolver.resolveName(java.lang.String,org.springframework.core.MethodParameter,org.springframework.web.context.request.NativeWebRequest)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "all",
-                "source": "P2",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "javax.naming.directory.DirContext.search(java.lang.String,java.lang.String,java.lang.Object[],javax.naming.directory.SearchControls)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P2",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "javax.naming.directory.DirContext.search(java.lang.String,java.lang.String,javax.naming.directory.SearchControls)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P2",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "javax.naming.directory.InitialDirContext.search(java.lang.String,java.lang.String,java.lang.Object[],javax.naming.directory.SearchControls)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P2",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "javax.naming.directory.InitialDirContext.search(java.lang.String,java.lang.String,javax.naming.directory.SearchControls)"
-            }
-        ],
-        "enable": 1,
-        "type": 4,
-        "value": "ldap-injection"
-    },
-    {
-        "details": [
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "source": "P1",
+                "inherit": "false",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [],
-                "target": "",
-                "track": "true",
+                "target": "R",
+                "track": "false",
                 "untags": [],
-                "value": "com.mongodb.DB.doEval(java.lang.String,java.lang.Object[])"
+                "value": "org.springframework.web.util.pattern.PathPattern.getPatternString()"
             }
         ],
         "enable": 1,
-        "type": 4,
-        "value": "nosql-injection"
+        "type": 2,
+        "value": "org.springframework.web.method.support.HandlerMethodArgumentResolver"
     },
     {
         "details": [
@@ -6428,40 +6272,52 @@
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
+                "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
-                "tags": [],
+                "tags": [
+                    "xml-encoded"
+                ],
                 "target": "R",
                 "track": "false",
-                "untags": [],
-                "value": "com.opensymphony.xwork2.util.ValueStack.findString(java.lang.String)"
+                "untags": [
+                    "xml-decoded"
+                ],
+                "value": "org.apache.taglibs.standard.util.EscapeXML.escape(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
+                "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
-                "tags": [],
+                "tags": [
+                    "html-encoded"
+                ],
                 "target": "R",
                 "track": "false",
-                "untags": [],
-                "value": "com.opensymphony.xwork2.util.ValueStack.findValue(java.lang.String)"
+                "untags": [
+                    "html-decoded"
+                ],
+                "value": "org.owasp.encoder.Encode.forHtml(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
+                "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
-                "tags": [],
+                "tags": [
+                    "html-encoded"
+                ],
                 "target": "R",
                 "track": "false",
-                "untags": [],
-                "value": "com.opensymphony.xwork2.util.ValueStack.findValue(java.lang.String,java.lang.Class)"
+                "untags": [
+                    "html-decoded"
+                ],
+                "value": "org.owasp.encoder.Encode.forHtmlAttribute(java.lang.String)"
             },
             {
                 "command": "",
@@ -6470,11 +6326,15 @@
                 "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
-                "tags": [],
+                "tags": [
+                    "html-encoded"
+                ],
                 "target": "R",
-                "track": "",
-                "untags": [],
-                "value": "ognl.Ognl.parseExpression(java.lang.String)"
+                "track": "false",
+                "untags": [
+                    "html-decoded"
+                ],
+                "value": "org.owasp.encoder.Encode.forHtmlContent(java.lang.String)"
             },
             {
                 "command": "",
@@ -6483,11 +6343,15 @@
                 "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
-                "tags": [],
-                "target": "O",
-                "track": "",
-                "untags": [],
-                "value": "ognl.OgnlParser.(java.io.InputStream)"
+                "tags": [
+                    "html-encoded"
+                ],
+                "target": "R",
+                "track": "false",
+                "untags": [
+                    "html-decoded"
+                ],
+                "value": "org.owasp.encoder.Encode.forHtmlUnquotedAttribute(java.lang.String)"
             },
             {
                 "command": "",
@@ -6496,11 +6360,15 @@
                 "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
-                "tags": [],
-                "target": "O",
-                "track": "",
-                "untags": [],
-                "value": "ognl.OgnlParser.(java.io.Reader)"
+                "tags": [
+                    "url-encoded"
+                ],
+                "target": "R",
+                "track": "false",
+                "untags": [
+                    "url-decoded"
+                ],
+                "value": "org.owasp.encoder.Encode.forUri(java.lang.String)"
             },
             {
                 "command": "",
@@ -6509,154 +6377,117 @@
                 "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
-                "tags": [],
-                "target": "O",
-                "track": "",
-                "untags": [],
-                "value": "ognl.OgnlParser.(ognl.OgnlParserTokenManager)"
+                "tags": [
+                    "url-encoded"
+                ],
+                "target": "R",
+                "track": "false",
+                "untags": [
+                    "url-decoded"
+                ],
+                "value": "org.owasp.encoder.Encode.forUriComponent(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "O",
+                "inherit": "true",
+                "source": "P1",
                 "stack_blacklist": [],
-                "tags": [],
+                "tags": [
+                    "xml-encoded"
+                ],
                 "target": "R",
-                "track": "",
-                "untags": [],
-                "value": "ognl.OgnlParser.topLevelExpression()"
-            }
-        ],
-        "enable": 1,
-        "type": 1,
-        "value": "ognl"
-    },
-    {
-        "details": [
+                "track": "false",
+                "untags": [
+                    "xml-decoded"
+                ],
+                "value": "org.owasp.encoder.Encode.forXml(java.lang.String)"
+            },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "P2",
+                "inherit": "true",
+                "source": "P1",
                 "stack_blacklist": [],
-                "tags": [],
+                "tags": [
+                    "xml-encoded"
+                ],
                 "target": "R",
                 "track": "false",
-                "untags": [],
-                "value": "com.squareup.okhttp.RequestBody.create(com.squareup.okhttp.MediaType,byte[],int,int)"
-            }
-        ],
-        "enable": 1,
-        "type": 1,
-        "value": "okhttp"
-    },
-    {
-        "details": [
+                "untags": [
+                    "xml-decoded"
+                ],
+                "value": "org.owasp.encoder.Encode.forXmlAttribute(java.lang.String)"
+            },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
-                "tags": [],
+                "tags": [
+                    "xml-encoded"
+                ],
                 "target": "R",
                 "track": "false",
-                "untags": [],
-                "value": "okhttp3.RequestBody$Companion.create(byte[],okhttp3.MediaType,int,int)"
+                "untags": [
+                    "xml-decoded"
+                ],
+                "value": "org.owasp.encoder.Encode.forXmlComment(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "P2",
+                "inherit": "true",
+                "source": "P1",
                 "stack_blacklist": [],
-                "tags": [],
+                "tags": [
+                    "xml-encoded"
+                ],
                 "target": "R",
                 "track": "false",
-                "untags": [],
-                "value": "okhttp3.RequestBody$Companion.create(okhttp3.MediaType,byte[],int,int)"
+                "untags": [
+                    "xml-decoded"
+                ],
+                "value": "org.owasp.encoder.Encode.forXmlContent(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "P2",
+                "inherit": "true",
+                "source": "P1",
                 "stack_blacklist": [],
-                "tags": [],
+                "tags": [
+                    "html-decoded"
+                ],
                 "target": "R",
                 "track": "false",
-                "untags": [],
-                "value": "okhttp3.RequestBody.create(okhttp3.MediaType,byte[],int,int)"
-            }
-        ],
-        "enable": 1,
-        "type": 1,
-        "value": "okhttp3"
-    },
-    {
-        "details": [
+                "untags": [
+                    "html-encoded"
+                ],
+                "value": "org.owasp.esapi.Encoder.decodeForHTML(java.lang.String)"
+            },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "O",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "R",
-                "track": "false",
-                "untags": [],
-                "value": "org.apache.commons.fileupload.FileItem.getString()"
-            }
-        ],
-        "enable": 1,
-        "type": 1,
-        "value": "org.apache.commons.fileupload.FileItem"
-    },
-    {
-        "details": [
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [
-                    "cross-site"
+                    "base64-decoded"
                 ],
                 "target": "R",
                 "track": "false",
-                "untags": [],
-                "value": "org.apache.commons.fileupload.FileUploadBase.parseRequest(org.apache.commons.fileupload.RequestContext)"
-            }
-        ],
-        "enable": 1,
-        "type": 2,
-        "value": "org.apache.commons.fileupload.FileUploadBase"
-    },
-    {
-        "details": [
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [
-                    "cross-site"
+                "untags": [
+                    "base64-encoded"
                 ],
-                "target": "R",
-                "track": "false",
-                "untags": [],
-                "value": "org.springframework.web.bind.annotation.support.HandlerMethodInvoker.resolvePathVariable(java.lang.String,org.springframework.core.MethodParameter,org.springframework.web.context.request.NativeWebRequest,java.lang.Object)"
+                "value": "org.owasp.esapi.Encoder.decodeFromBase64(java.lang.String)"
             },
             {
                 "command": "",
@@ -6666,76 +6497,55 @@
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [
-                    "cross-site"
-                ],
-                "target": "R",
-                "track": "false",
-                "untags": [],
-                "value": "org.springframework.web.servlet.mvc.method.annotation.PathVariableMethodArgumentResolver.resolveName(java.lang.String,org.springframework.core.MethodParameter,org.springframework.web.context.request.NativeWebRequest)"
-            }
-        ],
-        "enable": 1,
-        "type": 2,
-        "value": "org.springframework.web.method.support.HandlerMethodArgumentResolver"
-    },
-    {
-        "details": [
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [
-                    "xml-encoded"
+                    "url-decoded"
                 ],
                 "target": "R",
                 "track": "false",
                 "untags": [
-                    "xml-decoded"
+                    "url-encoded",
+                    "xss-encoded"
                 ],
-                "value": "org.apache.taglibs.standard.util.EscapeXML.escape(java.lang.String)"
+                "value": "org.owasp.esapi.Encoder.decodeFromURL(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [
-                    "html-encoded"
+                    "base64-encoded"
                 ],
                 "target": "R",
                 "track": "false",
                 "untags": [
-                    "html-decoded"
+                    "base64-decoded"
                 ],
-                "value": "org.owasp.encoder.Encode.forHtml(java.lang.String)"
+                "value": "org.owasp.esapi.Encoder.encodeForBase64(byte[],boolean)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [
-                    "html-encoded"
+                    "ldap-encoded"
                 ],
                 "target": "R",
                 "track": "false",
                 "untags": [
-                    "html-decoded"
+                    "ldap-decoded"
                 ],
-                "value": "org.owasp.encoder.Encode.forHtmlAttribute(java.lang.String)"
+                "value": "org.owasp.esapi.Encoder.encodeForDN(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [
@@ -6746,13 +6556,13 @@
                 "untags": [
                     "html-decoded"
                 ],
-                "value": "org.owasp.encoder.Encode.forHtmlContent(java.lang.String)"
+                "value": "org.owasp.esapi.Encoder.encodeForHTML(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [
@@ -6763,41 +6573,41 @@
                 "untags": [
                     "html-decoded"
                 ],
-                "value": "org.owasp.encoder.Encode.forHtmlUnquotedAttribute(java.lang.String)"
+                "value": "org.owasp.esapi.Encoder.encodeForHTMLAttribute(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [
-                    "url-encoded"
+                    "ldap-encoded"
                 ],
                 "target": "R",
                 "track": "false",
                 "untags": [
-                    "url-decoded"
+                    "ldap-decoded"
                 ],
-                "value": "org.owasp.encoder.Encode.forUri(java.lang.String)"
+                "value": "org.owasp.esapi.Encoder.encodeForLDAP(java.lang.String,boolean)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
+                "inherit": "true",
+                "source": "P2",
                 "stack_blacklist": [],
                 "tags": [
-                    "url-encoded"
+                    "sql-encoded"
                 ],
                 "target": "R",
                 "track": "false",
                 "untags": [
-                    "url-decoded"
+                    "sql-decoded"
                 ],
-                "value": "org.owasp.encoder.Encode.forUriComponent(java.lang.String)"
+                "value": "org.owasp.esapi.Encoder.encodeForSQL(org.owasp.esapi.codecs.Codec,java.lang.String)"
             },
             {
                 "command": "",
@@ -6807,14 +6617,14 @@
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [
-                    "xml-encoded"
+                    "url-encoded"
                 ],
                 "target": "R",
                 "track": "false",
                 "untags": [
-                    "xml-decoded"
+                    "url-decoded"
                 ],
-                "value": "org.owasp.encoder.Encode.forXml(java.lang.String)"
+                "value": "org.owasp.esapi.Encoder.encodeForURL(java.lang.String)"
             },
             {
                 "command": "",
@@ -6831,7 +6641,7 @@
                 "untags": [
                     "xml-decoded"
                 ],
-                "value": "org.owasp.encoder.Encode.forXmlAttribute(java.lang.String)"
+                "value": "org.owasp.esapi.Encoder.encodeForXML(java.lang.String)"
             },
             {
                 "command": "",
@@ -6848,7 +6658,7 @@
                 "untags": [
                     "xml-decoded"
                 ],
-                "value": "org.owasp.encoder.Encode.forXmlComment(java.lang.String)"
+                "value": "org.owasp.esapi.Encoder.encodeForXMLAttribute(java.lang.String)"
             },
             {
                 "command": "",
@@ -6858,37 +6668,37 @@
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [
-                    "xml-encoded"
+                    "xpath-encoded"
                 ],
                 "target": "R",
                 "track": "false",
                 "untags": [
-                    "xml-decoded"
+                    "xpath-decoded"
                 ],
-                "value": "org.owasp.encoder.Encode.forXmlContent(java.lang.String)"
+                "value": "org.owasp.esapi.Encoder.encodeForXPath(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
+                "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [
-                    "html-decoded"
+                    "base64-decoded"
                 ],
                 "target": "R",
                 "track": "false",
                 "untags": [
-                    "html-encoded"
+                    "base64-encoded"
                 ],
-                "value": "org.owasp.esapi.Encoder.decodeForHTML(java.lang.String)"
+                "value": "org.owasp.esapi.codecs.Base64.decode(byte[],int,int,int)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
+                "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [
@@ -6899,31 +6709,30 @@
                 "untags": [
                     "base64-encoded"
                 ],
-                "value": "org.owasp.esapi.Encoder.decodeFromBase64(java.lang.String)"
+                "value": "org.owasp.esapi.codecs.Base64.decode(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
+                "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [
-                    "url-decoded"
+                    "base64-decoded"
                 ],
                 "target": "R",
                 "track": "false",
                 "untags": [
-                    "url-encoded",
-                    "xss-encoded"
+                    "base64-encoded"
                 ],
-                "value": "org.owasp.esapi.Encoder.decodeFromURL(java.lang.String)"
+                "value": "org.owasp.esapi.codecs.Base64.decode(java.lang.String,int)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
+                "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [
@@ -6934,143 +6743,109 @@
                 "untags": [
                     "base64-decoded"
                 ],
-                "value": "org.owasp.esapi.Encoder.encodeForBase64(byte[],boolean)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [
-                    "ldap-encoded"
-                ],
-                "target": "R",
-                "track": "false",
-                "untags": [
-                    "ldap-decoded"
-                ],
-                "value": "org.owasp.esapi.Encoder.encodeForDN(java.lang.String)"
+                "value": "org.owasp.esapi.codecs.Base64.encodeBytes(byte[])"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
+                "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [
-                    "html-encoded"
+                    "base64-encoded"
                 ],
                 "target": "R",
                 "track": "false",
                 "untags": [
-                    "html-decoded"
+                    "base64-decoded"
                 ],
-                "value": "org.owasp.esapi.Encoder.encodeForHTML(java.lang.String)"
+                "value": "org.owasp.esapi.codecs.Base64.encodeBytes(byte[],int)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
+                "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [
-                    "html-encoded"
+                    "base64-encoded"
                 ],
                 "target": "R",
                 "track": "false",
                 "untags": [
-                    "html-decoded"
+                    "base64-decoded"
                 ],
-                "value": "org.owasp.esapi.Encoder.encodeForHTMLAttribute(java.lang.String)"
+                "value": "org.owasp.esapi.codecs.Base64.encodeBytes(byte[],int,int)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
+                "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [
-                    "ldap-encoded"
-                ],
-                "target": "R",
-                "track": "false",
-                "untags": [
-                    "ldap-decoded"
-                ],
-                "value": "org.owasp.esapi.Encoder.encodeForLDAP(java.lang.String,boolean)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "source": "P2",
-                "stack_blacklist": [],
-                "tags": [
-                    "sql-encoded"
+                    "base64-encoded"
                 ],
                 "target": "R",
                 "track": "false",
                 "untags": [
-                    "sql-decoded"
+                    "base64-decoded"
                 ],
-                "value": "org.owasp.esapi.Encoder.encodeForSQL(org.owasp.esapi.codecs.Codec,java.lang.String)"
+                "value": "org.owasp.esapi.codecs.Base64.encodeBytes(byte[],int,int,int)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
+                "inherit": "false",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [
-                    "url-encoded"
+                    "html-encoded"
                 ],
                 "target": "R",
                 "track": "false",
                 "untags": [
-                    "url-decoded"
+                    "html-decoded"
                 ],
-                "value": "org.owasp.esapi.Encoder.encodeForURL(java.lang.String)"
+                "value": "org.owasp.html.PolicyFactory.sanitize(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
-                "source": "P1",
+                "inherit": "false",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [
-                    "xml-encoded"
+                    "html-encoded"
                 ],
                 "target": "R",
                 "track": "false",
                 "untags": [
-                    "xml-decoded"
+                    "html-decoded"
                 ],
-                "value": "org.owasp.esapi.Encoder.encodeForXML(java.lang.String)"
+                "value": "org.owasp.validator.html.CleanResults.getCleanHTML()"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "true",
-                "source": "P1",
+                "inherit": "false",
+                "source": "O",
                 "stack_blacklist": [],
                 "tags": [
-                    "xml-encoded"
+                    "html-encoded"
                 ],
                 "target": "R",
                 "track": "false",
                 "untags": [
-                    "xml-decoded"
+                    "html-decoded"
                 ],
-                "value": "org.owasp.esapi.Encoder.encodeForXMLAttribute(java.lang.String)"
+                "value": "org.owasp.validator.html.CleanResults.getCleanXMLDocumentFragment()"
             },
             {
                 "command": "",
@@ -7080,932 +6855,164 @@
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [
-                    "xpath-encoded"
+                    "html-encoded"
                 ],
                 "target": "R",
                 "track": "false",
                 "untags": [
-                    "xpath-decoded"
-                ],
-                "value": "org.owasp.esapi.Encoder.encodeForXPath(java.lang.String)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [
-                    "base64-decoded"
-                ],
-                "target": "R",
-                "track": "false",
-                "untags": [
-                    "base64-encoded"
-                ],
-                "value": "org.owasp.esapi.codecs.Base64.decode(byte[],int,int,int)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [
-                    "base64-decoded"
-                ],
-                "target": "R",
-                "track": "false",
-                "untags": [
-                    "base64-encoded"
-                ],
-                "value": "org.owasp.esapi.codecs.Base64.decode(java.lang.String)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [
-                    "base64-decoded"
-                ],
-                "target": "R",
-                "track": "false",
-                "untags": [
-                    "base64-encoded"
-                ],
-                "value": "org.owasp.esapi.codecs.Base64.decode(java.lang.String,int)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [
-                    "base64-encoded"
-                ],
-                "target": "R",
-                "track": "false",
-                "untags": [
-                    "base64-decoded"
-                ],
-                "value": "org.owasp.esapi.codecs.Base64.encodeBytes(byte[])"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [
-                    "base64-encoded"
-                ],
-                "target": "R",
-                "track": "false",
-                "untags": [
-                    "base64-decoded"
-                ],
-                "value": "org.owasp.esapi.codecs.Base64.encodeBytes(byte[],int)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [
-                    "base64-encoded"
-                ],
-                "target": "R",
-                "track": "false",
-                "untags": [
-                    "base64-decoded"
-                ],
-                "value": "org.owasp.esapi.codecs.Base64.encodeBytes(byte[],int,int)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [
-                    "base64-encoded"
-                ],
-                "target": "R",
-                "track": "false",
-                "untags": [
-                    "base64-decoded"
-                ],
-                "value": "org.owasp.esapi.codecs.Base64.encodeBytes(byte[],int,int,int)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [
-                    "html-encoded"
-                ],
-                "target": "R",
-                "track": "false",
-                "untags": [
-                    "html-decoded"
-                ],
-                "value": "org.owasp.html.PolicyFactory.sanitize(java.lang.String)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "O",
-                "stack_blacklist": [],
-                "tags": [
-                    "html-encoded"
-                ],
-                "target": "R",
-                "track": "false",
-                "untags": [
-                    "html-decoded"
-                ],
-                "value": "org.owasp.validator.html.CleanResults.getCleanHTML()"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "O",
-                "stack_blacklist": [],
-                "tags": [
-                    "html-encoded"
-                ],
-                "target": "R",
-                "track": "false",
-                "untags": [
-                    "html-decoded"
-                ],
-                "value": "org.owasp.validator.html.CleanResults.getCleanXMLDocumentFragment()"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [
-                    "html-encoded"
-                ],
-                "target": "R",
-                "track": "false",
-                "untags": [
-                    "html-decoded"
+                    "html-decoded"
                 ],
                 "value": "org.owasp.validator.html.scan.AbstractAntiSamyScanner.scan(java.lang.String)"
             }
         ],
-        "enable": 1,
-        "type": 1,
-        "value": "owasp-esapi"
-    },
-    {
-        "details": [
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P2",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "java.io.File.(java.io.File,java.lang.String)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [
-                    "org.owasp.esapi.reference.DefaultValidator.isValidDirectoryPath",
-                    "org.owasp.esapi.reference.DefaultValidator.isValidFileName"
-                ],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "java.io.File.(java.lang.String)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1,2",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "java.io.File.(java.lang.String,java.lang.String)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "java.io.File.(java.net.URI)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "source": "P1,2",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "java.io.File.createTempFile(java.lang.String,java.lang.String,java.io.File)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "java.io.FileInputStream.(java.lang.String)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1,2",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "java.nio.file.FileSystem.getPath(java.lang.String,java.lang.String[])"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "java.nio.file.spi.FileSystemProvider.getFileSystem(java.net.URI)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "java.nio.file.spi.FileSystemProvider.newFileSystem(java.net.URI,java.util.Map)"
-            }
-        ],
-        "enable": 1,
-        "type": 4,
-        "value": "path-traversal"
-    },
-    {
-        "details": [
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
-                "stack_blacklist": [
-                    "org.springframework.web.util.UriComponentsBuilder.fromOriginHeader",
-                    "org.springframework.web.util.UriComponentsBuilder.fromUriString"
-                ],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "java.util.regex.Pattern.matcher(java.lang.CharSequence)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "jregex.Pattern.matcher(java.lang.String)"
-            }
-        ],
-        "enable": 1,
-        "type": 4,
-        "value": "redos"
-    },
-    {
-        "details": [
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P2",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "com.github.mustachejava.codes.ValueCode.execute(java.io.Writer,java.lang.String)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "com.sun.faces.renderkit.html_basic.HtmlResponseWriter.write(java.lang.String)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "jakarta.servlet.ServletOutputStream.print(java.lang.String)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "jakarta.servlet.ServletOutputStream.println(java.lang.String)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "jakarta.servlet.ServletOutputStream.write(byte[])"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "jakarta.servlet.ServletOutputStream.write(byte[],int,int)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1,2",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "java.io.PrintWriter.format(java.lang.String,java.lang.Object[])"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P2,3",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "java.io.PrintWriter.format(java.util.Locale,java.lang.String,java.lang.Object[])"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "java.io.PrintWriter.print(char[])"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "java.io.PrintWriter.print(java.lang.Object)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "java.io.PrintWriter.print(java.lang.String)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1,2",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "java.io.PrintWriter.printf(java.lang.String,java.lang.Object[])"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P2,3",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "java.io.PrintWriter.printf(java.util.Locale,java.lang.String,java.lang.Object[])"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "java.io.PrintWriter.println(char[])"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "java.io.PrintWriter.println(java.lang.Object)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "java.io.PrintWriter.println(java.lang.String)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "java.io.PrintWriter.write(char[])"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "java.io.PrintWriter.write(char[],int,int)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "java.io.PrintWriter.write(java.lang.String)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "java.io.PrintWriter.write(java.lang.String,int,int)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "javax.servlet.ServletOutputStream.print(java.lang.String)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "javax.servlet.ServletOutputStream.println(java.lang.String)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "javax.servlet.ServletOutputStream.write(byte[])"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "javax.servlet.ServletOutputStream.write(byte[],int,int)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "javax.servlet.jsp.JspWriter.print(java.lang.String)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "javax.servlet.jsp.JspWriter.println(java.lang.String)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "javax.servlet.jsp.JspWriter.write(char[])"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [
-                    "OutSupport.writeEscapedXml"
-                ],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "javax.servlet.jsp.JspWriter.write(char[],int,int)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "javax.servlet.jsp.JspWriter.write(java.lang.String)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "javax.servlet.jsp.JspWriter.write(java.lang.String,int,int)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "org.apache.tapestry5.internal.services.MarkupWriterImpl.writeRaw(java.lang.String)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "org.glassfish.jersey.message.internal.AbstractMessageReaderWriterProvider.writeToAsString(java.lang.String,java.io.OutputStream,javax.ws.rs.core.MediaType)"
-            },
+        "enable": 1,
+        "type": 1,
+        "value": "owasp-esapi"
+    },
+    {
+        "details": [
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
+                "inherit": "all",
+                "source": "P2",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.glassfish.jersey.message.internal.ByteArrayProvider.writeTo(byte[],java.lang.Class,java.lang.reflect.Type,java.lang.annotation.Annotation[],javax.ws.rs.core.MediaType,javax.ws.rs.core.MultivaluedMap,java.io.OutputStream)"
+                "value": "java.io.File.(java.io.File,java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "all",
                 "source": "P1",
-                "stack_blacklist": [],
+                "stack_blacklist": [
+                    "org.owasp.esapi.reference.DefaultValidator.isValidDirectoryPath",
+                    "org.owasp.esapi.reference.DefaultValidator.isValidFileName"
+                ],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.glassfish.jersey.message.internal.DataSourceProvider.writeTo(javax.activation.DataSource,java.lang.Class,java.lang.reflect.Type,java.lang.annotation.Annotation[],javax.ws.rs.core.MediaType,javax.ws.rs.core.MultivaluedMap,java.io.OutputStream)"
+                "value": "java.io.File.(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
+                "inherit": "all",
+                "source": "P1,2",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.glassfish.jersey.message.internal.FileProvider.writeTo(java.io.File,java.lang.Class,java.lang.reflect.Type,java.lang.annotation.Annotation[],javax.ws.rs.core.MediaType,javax.ws.rs.core.MultivaluedMap,java.io.OutputStream)"
+                "value": "java.io.File.(java.lang.String,java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "all",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.glassfish.jersey.message.internal.InputStreamProvider.writeTo(java.io.InputStream,java.lang.Class,java.lang.reflect.Type,java.lang.annotation.Annotation[],javax.ws.rs.core.MediaType,javax.ws.rs.core.MultivaluedMap,java.io.OutputStream)"
+                "value": "java.io.File.(java.net.URI)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
+                "inherit": "true",
+                "source": "P1,2",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.glassfish.jersey.message.internal.ReaderProvider.writeTo(java.io.Reader,java.lang.Class,java.lang.reflect.Type,java.lang.annotation.Annotation[],javax.ws.rs.core.MediaType,javax.ws.rs.core.MultivaluedMap,java.io.OutputStream)"
+                "value": "java.io.File.createTempFile(java.lang.String,java.lang.String,java.io.File)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.glassfish.jersey.message.internal.StringMessageProvider.writeTo(java.lang.String,java.lang.Class,java.lang.reflect.Type,java.lang.annotation.Annotation[],javax.ws.rs.core.MediaType,javax.ws.rs.core.MultivaluedMap,java.io.OutputStream)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P3",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "org.mule.module.http.internal.domain.response.DefaultHttpResponse.(org.mule.module.http.internal.domain.response.ResponseStatus,org.apache.commons.collections.MultiMap,org.mule.module.http.internal.domain.HttpEntity)"
+                "value": "java.io.FileInputStream.(java.lang.String)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1",
+                "inherit": "all",
+                "source": "P1,2",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.mule.module.http.internal.domain.response.HttpResponseBuilder.build(org.mule.module.http.internal.domain.HttpEntity)"
+                "value": "java.nio.file.FileSystem.getPath(java.lang.String,java.lang.String[])"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.mule.module.http.internal.domain.response.HttpResponseBuilder.setEntity(org.mule.module.http.internal.domain.HttpEntity)"
+                "value": "java.nio.file.spi.FileSystemProvider.getFileSystem(java.net.URI)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
-                "inherit": "false",
+                "inherit": "true",
                 "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.mule.transformer.simple.SetPayloadMessageProcessor.resolveDataType(org.mule.DefaultMuleEvent,java.lang.Object)"
-            },
+                "value": "java.nio.file.spi.FileSystemProvider.newFileSystem(java.net.URI,java.util.Map)"
+            }
+        ],
+        "enable": 1,
+        "type": 4,
+        "value": "path-traversal"
+    },
+    {
+        "details": [
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "false",
                 "source": "P1",
-                "stack_blacklist": [],
+                "stack_blacklist": [
+                    "org.springframework.web.util.UriComponentsBuilder.fromOriginHeader",
+                    "org.springframework.web.util.UriComponentsBuilder.fromUriString"
+                ],
                 "tags": [],
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.mule.transformer.simple.SetPayloadMessageProcessor.resolveDataType(org.mule.api.MuleEvent,java.lang.Object)"
+                "value": "java.util.regex.Pattern.matcher(java.lang.CharSequence)"
             },
             {
                 "command": "",
@@ -8018,12 +7025,12 @@
                 "target": "",
                 "track": "true",
                 "untags": [],
-                "value": "org.springframework.http.converter.StringHttpMessageConverter.writeInternal(java.lang.String,org.springframework.http.HttpOutputMessage)"
+                "value": "jregex.Pattern.matcher(java.lang.String)"
             }
         ],
         "enable": 1,
         "type": 4,
-        "value": "reflected-xss"
+        "value": "redos"
     },
     {
         "details": [
@@ -8460,6 +7467,32 @@
                 "untags": [],
                 "value": "org.springframework.cloud.config.server.resource.ResourceController.retrieve(java.lang.String,java.lang.String,java.lang.String,java.lang.String,boolean)"
             },
+            {
+                "command": "",
+                "ignore_blacklist": false,
+                "ignore_internal": false,
+                "inherit": "all",
+                "source": "P1",
+                "stack_blacklist": [],
+                "tags": [],
+                "target": "R",
+                "track": "false",
+                "untags": [],
+                "value": "org.springframework.http.server.ServletServerHttpRequest.getBodyFromServletRequestParameters(jakarta.servlet.http.HttpServletRequest)"
+            },
+            {
+                "command": "",
+                "ignore_blacklist": false,
+                "ignore_internal": false,
+                "inherit": "all",
+                "source": "P1",
+                "stack_blacklist": [],
+                "tags": [],
+                "target": "R",
+                "track": "false",
+                "untags": [],
+                "value": "org.springframework.http.server.ServletServerHttpRequest.getBodyFromServletRequestParameters(javax.servlet.http.HttpServletRequest)"
+            },
             {
                 "command": "",
                 "ignore_blacklist": false,
@@ -9468,104 +8501,6 @@
         "type": 4,
         "value": "ssrf"
     },
-    {
-        "details": [
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1,2",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "coldfusion.runtime.SessionScope.bind(java.lang.String,java.lang.Object)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "source": "P1,2",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "jakarta.servlet.http.HttpSession.putValue(java.lang.String,java.lang.Object)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "source": "P1,2",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "jakarta.servlet.http.HttpSession.setAttribute(java.lang.String,java.lang.Object)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "source": "P1,2",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "javax.servlet.http.HttpSession.putValue(java.lang.String,java.lang.Object)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "source": "P1,2",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "javax.servlet.http.HttpSession.setAttribute(java.lang.String,java.lang.Object)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1,2",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "org.apache.struts2.dispatcher.SessionMap.put(java.lang.Object,java.lang.Object)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1,2",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "play.mvc.Http$Session.put(java.lang.String,java.lang.String)"
-            }
-        ],
-        "enable": 1,
-        "type": 4,
-        "value": "trust-boundary-violation"
-    },
     {
         "details": [
             {
@@ -10722,6 +9657,19 @@
                 "untags": [],
                 "value": "jakarta.xml.bind.Unmarshaller.unmarshal(org.xml.sax.InputSource)"
             },
+            {
+                "command": "",
+                "ignore_blacklist": false,
+                "ignore_internal": false,
+                "inherit": "true",
+                "source": "P1",
+                "stack_blacklist": [],
+                "tags": [],
+                "target": "",
+                "track": "true",
+                "untags": [],
+                "value": "jakarta.xml.parsers.DocumentBuilder.parse(java.io.File)"
+            },
             {
                 "command": "",
                 "ignore_blacklist": false,
@@ -11217,40 +10165,27 @@
                 "track": "true",
                 "untags": [],
                 "value": "nu.xom.Builder.build(java.lang.String)"
-            }
-        ],
-        "enable": 1,
-        "type": 4,
-        "value": "xxe"
-    },
-    {
-        "details": [
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "source": "",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "",
-                "untags": [],
-                "value": "com.sun.org.apache.xerces.internal.xni.grammars.XMLGrammarLoader.setFeature(java.lang.String, boolean)"
             },
             {
                 "command": "",
                 "ignore_blacklist": false,
                 "ignore_internal": false,
                 "inherit": "true",
-                "source": "",
+                "source": "P1",
                 "stack_blacklist": [],
                 "tags": [],
                 "target": "",
-                "track": "",
+                "track": "true",
                 "untags": [],
-                "value": "com.sun.org.apache.xerces.internal.xni.parser.XMLParserConfiguration.setFeature(java.lang.String, boolean)"
-            },
+                "value": "org.xml.sax.XMLReader.parse(org.xml.sax.InputSource)"
+            }
+        ],
+        "enable": 1,
+        "type": 4,
+        "value": "xxe"
+    },
+    {
+        "details": [
             {
                 "command": "",
                 "ignore_blacklist": false,
@@ -11276,32 +10211,6 @@
                 "track": "",
                 "untags": [],
                 "value": "javax.xml.parsers.SAXParserFactory.setFeature(java.lang.String, boolean)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "source": "",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "",
-                "untags": [],
-                "value": "javax.xml.transform.TransformerFactory.setFeature(java.lang.String, boolean)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "source": "",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "",
-                "untags": [],
-                "value": "org.xml.sax.XMLReader.setFeature(java.lang.String, boolean)"
             }
         ],
         "enable": 1,
diff --git a/static/data/python_full_policy.json b/static/data/python_full_policy.json
index eff2a626d..92610e536 100644
--- a/static/data/python_full_policy.json
+++ b/static/data/python_full_policy.json
@@ -358,7 +358,7 @@
                 "value": "Cryptodome.Cipher._mode_openpgp.OpenPgpMode.encrypt"
             }
         ],
-        "enable": 1,
+        "enable": 0,
         "type": 4,
         "value": "crypto-bad-ciphers"
     },
@@ -1854,7 +1854,7 @@
                 "value": "flask.app.Flask.make_response"
             }
         ],
-        "enable": 1,
+        "enable": 0,
         "type": 4,
         "value": "reflected-xss"
     },
diff --git a/static/data/python_policy.json b/static/data/python_policy.json
index e7135ac95..380113cba 100644
--- a/static/data/python_policy.json
+++ b/static/data/python_policy.json
@@ -78,195 +78,6 @@
         "type": 4,
         "value": "code-execution"
     },
-    {
-        "details": [
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1,2,plaintext",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "Crypto.Cipher._mode_cbc.CbcMode.encrypt"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1,2,plaintext",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "Crypto.Cipher._mode_cfb.CfbMode.encrypt"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1,2,plaintext",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "Crypto.Cipher._mode_ctr.CtrMode.encrypt"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1,2,plaintext",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "Crypto.Cipher._mode_eax.EaxMode.encrypt"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1,2,plaintext",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "Crypto.Cipher._mode_ecb.EcbMode.encrypt"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1,2,plaintext",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "Crypto.Cipher._mode_ofb.OfbMode.encrypt"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1,2,plaintext",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "Crypto.Cipher._mode_openpgp.OpenPgpMode.encrypt"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1,2,plaintext",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "Cryptodome.Cipher._mode_cbc.CbcMode.encrypt"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1,2,plaintext",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "Cryptodome.Cipher._mode_cfb.CfbMode.encrypt"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1,2,plaintext",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "Cryptodome.Cipher._mode_ctr.CtrMode.encrypt"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1,2,plaintext",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "Cryptodome.Cipher._mode_eax.EaxMode.encrypt"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1,2,plaintext",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "Cryptodome.Cipher._mode_ecb.EcbMode.encrypt"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1,2,plaintext",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "Cryptodome.Cipher._mode_ofb.OfbMode.encrypt"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P1,2,plaintext",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "Cryptodome.Cipher._mode_openpgp.OpenPgpMode.encrypt"
-            }
-        ],
-        "enable": 1,
-        "type": 4,
-        "value": "crypto-bad-ciphers"
-    },
     {
         "details": [
             {
@@ -1330,39 +1141,6 @@
         "type": 4,
         "value": "redos"
     },
-    {
-        "details": [
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P2,content",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "django.http.response.HttpResponse.__init__"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "false",
-                "source": "P2,rv",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "true",
-                "untags": [],
-                "value": "flask.app.Flask.make_response"
-            }
-        ],
-        "enable": 1,
-        "type": 4,
-        "value": "reflected-xss"
-    },
     {
         "details": [
             {
diff --git a/static/data/vul_strategy.json b/static/data/vul_strategy.json
index 00460f796..3f52604ab 100644
--- a/static/data/vul_strategy.json
+++ b/static/data/vul_strategy.json
@@ -145,7 +145,7 @@
     },
     {
         "level": 3,
-        "state": "enable",
+        "state": "disable",
         "system_type": 1,
         "user": 1,
         "vul_desc": "\u6ca1\u6709\u8bbe\u7f6eHTTPS\u4f1a\u8bdd\u4e2d\u654f\u611fcookie\u7684\u5b89\u5168\u5c5e\u6027\uff0c\u8fd9\u53ef\u80fd\u5bfc\u81f4\u7528\u6237\u4ee3\u7406\u901a\u8fc7HTTP\u4f1a\u8bdd\u4ee5\u660e\u6587\u53d1\u9001\u8fd9\u4e9bcookie\u3002",
@@ -161,7 +161,7 @@
     },
     {
         "level": 3,
-        "state": "enable",
+        "state": "disable",
         "system_type": 1,
         "user": 1,
         "vul_desc": "\u6570\u636e\u901a\u8fc7\u5f31\u52a0\u5bc6\u7b97\u6cd5MessageDigest\u51fd\u6570\u8fdb\u884c\u52a0\u5bc6",
@@ -177,7 +177,7 @@
     },
     {
         "level": 3,
-        "state": "enable",
+        "state": "disable",
         "system_type": 1,
         "user": 1,
         "vul_desc": "\u54c8\u5e0c\u7b97\u6cd5\u662f\u4f7f\u7528\u54c8\u5e0c\u51fd\u6570\u5c06\u4efb\u610f\u957f\u5ea6\u7684\u6d88\u606f\u6620\u5c04\u6210\u4e3a\u4e00\u4e2a\u957f\u5ea6\u8f83\u77ed\u4e14\u957f\u5ea6\u56fa\u5b9a\u7684\u503c\uff0c\u8fd9\u4e2a\u7ecf\u8fc7\u6620\u5c04\u7684\u503c\u4e3a\u54c8\u5e0c\u503c\u3002\u5b83\u662f\u4e00\u79cd\u5355\u5411\u52a0\u5bc6\u4f53\u5236\uff0c\u5373\u4e00\u4e2a\u4ece\u660e\u6587\u5230\u5bc6\u6587\u7684\u4e0d\u53ef\u9006\u6620\u5c04\uff0c\u53ea\u6709\u52a0\u5bc6\u8fc7\u7a0b\uff0c\u6ca1\u6709\u89e3\u5bc6\u8fc7\u7a0b\u3002\u800c\u4e0d\u5b89\u5168\u7684\u54c8\u5e0c\u7b97\u6cd5\u5219\u53ef\u4ee5\u9006\u5411\u63a8\u51fa\u660e\u6587\u3002\u5728\u5bc6\u7801\u5b66\u4e2d\uff0c\u54c8\u5e0c\u7b97\u6cd5\u4e3b\u8981\u7528\u4e8e\u6d88\u606f\u6458\u8981\u548c\u7b7e\u540d\u6765\u5bf9\u6574\u4e2a\u6d88\u606f\u7684\u5b8c\u6574\u6027\u8fdb\u884c\u6821\u9a8c\uff0c\u6240\u4ee5\u9700\u8981\u54c8\u5e0c\u7b97\u6cd5\u65e0\u6cd5\u63a8\u5bfc\u8f93\u5165\u7684\u539f\u59cb\u503c\uff0c\u8fd9\u662f\u54c8\u5e0c\u7b97\u6cd5\u5b89\u5168\u6027\u7684\u57fa\u7840\u3002\u76ee\u524d\u5e38\u7528\u7684\u54c8\u5e0c\u7b97\u6cd5\u5305\u62ecMD4\u3001MD5\u3001SHA\u7b49\u3002\u672c\u7bc7\u6587\u7ae0\u4ee5JAVA\u8bed\u8a00\u6e90\u4ee3\u7801\u4e3a\u4f8b\uff0c\u5206\u6790\u4e0d\u5b89\u5168\u7684\u54c8\u5e0c\u7b97\u6cd5\u7f3a\u9677\u4ea7\u751f\u7684\u539f\u56e0\u4ee5\u53ca\u4fee\u590d\u65b9\u6cd5\u3002\u8be6\u7ec6\u8bf7\u53c2\u89c1\uff1aCWE ID 327: Use of a Broken or Risky Cryptographic Algorithm (http://cwe.mitre.org/data/definitions/327.html)\u3002",
@@ -193,7 +193,7 @@
     },
     {
         "level": 3,
-        "state": "enable",
+        "state": "disable",
         "system_type": 1,
         "user": 1,
         "vul_desc": "\u968f\u673a\u6570\u5728\u8ba1\u7b97\u673a\u5e94\u7528\u4e2d\u4f7f\u7528\u7684\u6bd4\u8f83\u5e7f\u6cdb\uff0c\u6700\u4e3a\u719f\u77e5\u7684\u4fbf\u662f\u5728\u5bc6\u7801\u5b66\u4e2d\u7684\u5e94\u7528\u3002\u968f\u673a\u6570\u5206\u4e3a\u771f\u968f\u673a\u6570\u548c\u4f2a\u968f\u673a\u6570\uff0c\u6211\u4eec\u7a0b\u5e8f\u4f7f\u7528\u7684\u57fa\u672c\u90fd\u662f\u4f2a\u968f\u673a\u6570\u3002\u4f2a\u968f\u673a\u53c8\u5206\u4e3a\u5f3a\u4f2a\u968f\u673a\u6570\u548c\u5f31\u4f2a\u968f\u673a\u6570\u3002\u4f2a\u968f\u673a\u6570\uff0c\u901a\u8fc7\u4e00\u5b9a\u7b97\u6cd5\u548c\u79cd\u5b50\u5f97\u51fa\u3002\u8f6f\u4ef6\u5b9e\u73b0\u7684\u662f\u4f2a\u968f\u673a\u6570\u3002\u5f3a\u4f2a\u968f\u673a\u6570\uff0c\u96be\u4ee5\u9884\u6d4b\u7684\u968f\u673a\u6570\u3002\u5f31\u4f2a\u968f\u673a\u6570\uff0c\u6613\u4e8e\u9884\u6d4b\u7684\u968f\u673a\u6570\u3002\nJava\u7a0b\u5e8f\u4e2d\uff0c\u4f7f\u7528java.util.Random\u83b7\u5f97\u968f\u673a\u6570\uff0c\u8fd9\u79cd\u968f\u673a\u6570\u6e90\u4e8e\u4f2a\u968f\u673a\u6570\u751f\u6210\u5668\uff0c\u4ea7\u751f\u7684\u968f\u673a\u6570\u5bb9\u6613\u88ab\u9884\u6d4b\uff0c\u5bf9\u4e8e\u5b89\u5168\u6027\u8981\u6c42\u8f83\u9ad8\u7684\u73af\u5883\u4e2d\uff0c\u4f7f\u7528\u8fd9\u79cd\u968f\u673a\u6570\u53ef\u80fd\u4f1a\u964d\u4f4e\u7cfb\u7edf\u5b89\u5168\u6027\uff0c\u4f7f\u653b\u51fb\u8005\u6709\u673a\u53ef\u4e58\u3002",
@@ -337,7 +337,7 @@
     },
     {
         "level": 2,
-        "state": "enable",
+        "state": "disable",
         "system_type": 1,
         "user": 1,
         "vul_desc": "\u8de8\u7ad9\u811a\u672c\u653b\u51fb\u6f0f\u6d1e\u7b80\u79f0XSS\u6f0f\u6d1e,\u4e3b\u8981\u662f\u7531\u4e8e\u5e94\u7528\u540e\u7aef\u672a\u5bf9\u7528\u6237\u8f93\u5165\u8fdb\u884c\u5b89\u5168\u6821\u9a8c\u6216\u6821\u9a8c\u4e0d\u4e25\u683c\u5bfc\u81f4\u6076\u610f\u7528\u6237\u53ef\u81ea\u5b9a\u4e49\u63a7\u5236\u9875\u9762\u8f93\u51fa\u5185\u5bb9\uff0c\u4ece\u800c\u4ea7\u751f\u8de8\u7ad9\u811a\u672c\u653b\u51fb\u6f0f\u6d1e\u3002\u901a\u5e38\u6765\u8bf4\u6076\u610f\u7528\u6237\u901a\u8fc7\u6784\u9020\u95ed\u5408\u6807\u7b7e\u65b9\u5f0f\u5728HTML\u9875\u9762\u4e2d\u63d2\u5165Javascript\u4ee3\u7801\uff0c\u5728\u6b63\u5e38\u7528\u6237\u6d4f\u89c8\u6b64\u9875\u9762\u65f6\u5bf9\u6b63\u5e38\u7528\u6237\u8fdb\u884c\u653b\u51fb\uff0c\u5e38\u89c1\u7684\u653b\u51fb\u65b9\u5f0f\u6709\u83b7\u53d6\u7528\u6237\u7684\u8eab\u4efd\u51ed\u636e\u3001\u5bf9\u7528\u6237\u5185\u7f51\u8fdb\u884c\u63a2\u6d4b\u626b\u63cf\u3001\u6267\u884c\u9493\u9c7c\u6b3a\u9a97\u653b\u51fb\u7b49",
@@ -417,7 +417,7 @@
     },
     {
         "level": 3,
-        "state": "enable",
+        "state": "disable",
         "system_type": 1,
         "user": 1,
         "vul_desc": "CWE-501\uff0c\u4fe1\u4efb\u8fb9\u754c\u53ef\u4ee5\u8ba4\u4e3a\u662f\u901a\u8fc7\u7a0b\u5e8f\u7ed8\u5236\u7684\u7ebf\u3002 \u5728\u751f\u4ea7\u7ebf\u7684\u4e00\u4fa7\uff0c\u6570\u636e\u4e0d\u53d7\u4fe1\u4efb\u3002 \u5728\u8be5\u884c\u7684\u53e6\u4e00\u7aef\uff0c\u5047\u5b9a\u6570\u636e\u662f\u53ef\u4fe1\u7684\u3002 \u9a8c\u8bc1\u903b\u8f91\u7684\u76ee\u7684\u662f\u5141\u8bb8\u6570\u636e\u5b89\u5168\u5730\u8d8a\u8fc7\u4fe1\u4efb\u8fb9\u754c-\u4ece\u4e0d\u53d7\u4fe1\u4efb\u53d8\u4e3a\u53d7\u4fe1\u4efb\u3002 \u5f53\u7a0b\u5e8f\u6a21\u7cca\u4e86\u53ef\u4fe1\u548c\u4e0d\u53ef\u4fe1\u4e4b\u95f4\u7684\u754c\u9650\u65f6\uff0c\u5c31\u4f1a\u53d1\u751f\u4fe1\u4efb\u8fb9\u754c\u51b2\u7a81\u3002 \u901a\u8fc7\u5728\u540c\u4e00\u6570\u636e\u7ed3\u6784\u4e2d\u7ec4\u5408\u53ef\u4fe1\u6570\u636e\u548c\u4e0d\u53ef\u4fe1\u6570\u636e\uff0c\u7a0b\u5e8f\u5458\u53ef\u4ee5\u66f4\u5bb9\u6613\u5730\u9519\u8bef\u5730\u4fe1\u4efb\u672a\u9a8c\u8bc1\u7684\u6570\u636e\u3002",

From 5525f8fd1a8493d88379eb2437510e2d96d3ccbd Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Thu, 13 Jul 2023 12:06:59 +0800
Subject: [PATCH 134/161] feat: update 1.13.0 params.

---
 static/data/java_params.json | 200 ++++++++++++++---------------------
 1 file changed, 78 insertions(+), 122 deletions(-)

diff --git a/static/data/java_params.json b/static/data/java_params.json
index 4510fa899..5bc587616 100644
--- a/static/data/java_params.json
+++ b/static/data/java_params.json
@@ -1,122 +1,78 @@
-      {
-						"org.springframework.web.method.support.HandlerMethodArgumentResolver.resolveArgument": [
-                    "GET",
-                    "POST",
-                    "HEADER",
-                    "PATH"
-           ],
- 					"org.springframework.web.method.annotation.ServletModelAttributeMethodProcessor.resolveArgument": [
-                    "GET",
-                    "POST",
-                    "HEADER",
-                    "PATH"
-           ],
-						"org.springframework.web.servlet.mvc.method.annotation.PathVariableMethodArgumentResolver.resolveName":["PATH"],
-						"org.springframework.web.bind.annotation.support.HandlerMethodInvoker.resolvePathVariable":["PATH"],
-            "javax.servlet.ServletRequest.getParameter": [
-                    "GET",
-                    "POST"
-            ],
-            "javax.servlet.ServletRequest.getParameterValues": [
-                    "GET",
-                    "POST"
-            ],
-            "javax.servlet.http.HttpServletRequest.getHeader": ["HEADER"],
-            "javax.servlet.http.HttpServletRequest.getQueryString": ["GET"],
-            "javax.servlet.http.HttpServletRequest.getCookies": ["COOKIE"],
-            "io.grpc.MethodDescriptor.parseRequest":["GRPC"],
-            "org.apache.struts2.dispatcher.multipart.MultiPartRequest.getParameterValues":[
-                    "GET",
-                    "POST"
-            ],
-            "javax.servlet.ServletRequest.getInputStream":[
-                    "GET",
-                    "POST"
-            ],
-            "javax.servlet.ServletRequest.getParameterNames()":[
-                    "GET",
-                    "POST"
-            ],
-            "javax.servlet.ServletRequest.getParameterMap":[
-                    "GET",
-                    "POST"
-            ],
-            "javax.servlet.ServletRequest.getReader":[
-                    "GET",
-                    "POST"
-            ],
-            "javax.servlet.http.HttpServletRequest.getHeaderNames()":["HEADER"],
-            "javax.servlet.http.HttpServletRequest.getParts":[
-                    "GET",
-                    "POST"
-            ],
-            "javax.servlet.http.HttpServletRequest.getPart":[
-                    "GET",
-                    "POST"
-            ],
-            "javax.servlet.http.HttpServletRequest.getInputStream":[
-                    "GET",
-                    "POST"
-            ],
-            "javax.servlet.http.HttpServletRequest.getHeader":["HEADER"],
-            "javax.servlet.http.HttpServletRequest.getCookies":["COOKIE"],
-            "javax.servlet.http.HttpServletRequest.getHeaders":["HEADER"],
-            "javax.servlet.http.HttpServletRequest.getQueryString":["GET"],
-            "javax.servlet.http.HttpServletRequest.getParameter":[
-                    "GET",
-                    "POST"
-            ],
-            "javax.servlet.http.HttpServletRequest.getParameterValues":[
-                    "GET",
-                    "POST"
-            ],
-            "javax.servlet.http.HttpServletRequest.getParameterNames()":[
-                    "GET",
-                    "POST"
-            ],
-            "javax.servlet.http.HttpServletRequest.getParameterMap()":[
-                    "GET",
-                    "POST"
-            ],
-            "javax.servlet.http.HttpServletRequest.getReader()":[
-                    "GET",
-                    "POST"
-            ],
-            "org.apache.commons.fileupload.FileUploadBase.parseRequest(org.apache.commons.fileupload.RequestContext)":[
-                    "POST"
-            ],
-            "jakarta.servlet.ServletRequest.getInputStream":[
-                    "GET",
-                    "POST"
-            ],
-            "jakarta.servlet.ServletRequest.getParameter":[
-                    "GET",
-                    "POST"
-            ],
-            "jakarta.servlet.ServletRequest.getParameterNames":[
-                    "GET",
-                    "POST"
-            ],
-            "jakarta.servlet.ServletRequest.getParameterValues":[
-                    "GET",
-                    "POST"
-            ],
-            "jakarta.servlet.ServletRequest.getReader":[
-                    "GET",
-                    "POST"
-            ],
-            "jakarta.servlet.http.HttpServletRequest.getHeader":["HEADER"],
-            "jakarta.servlet.http.HttpServletRequest.getHeaders":["HEADER"],
-            "jakarta.servlet.http.HttpServletRequest.getHeaderNames":["HEADER"],
-            "jakarta.servlet.http.HttpServletRequest.getParts":[
-                    "GET",
-                    "POST"
-            ],
-            "jakarta.servlet.http.HttpServletRequest.getPart":[
-                    "GET",
-                    "POST"
-            ],
-            "jakarta.servlet.http.HttpServletRequest.getCookies":["COOKIE"],
-            "jakarta.servlet.http.HttpServletRequest.getQueryString":["GET"],
-            "javax.servlet.http.HttpServletRequest.getSession":["SESSION"]
-    }
+{
+	"org.springframework.web.method.support.HandlerMethodArgumentResolver.resolveArgument": ["GET", "POST", "HEADER", "PATH"],
+	"org.springframework.web.method.annotation.ServletModelAttributeMethodProcessor.resolveArgument": ["GET", "POST", "HEADER", "PATH"],
+	"org.springframework.web.servlet.mvc.method.annotation.PathVariableMethodArgumentResolver.resolveName": ["PATH"],
+	"org.springframework.web.bind.annotation.support.HandlerMethodInvoker.resolvePathVariable": ["PATH"],
+	"javax.servlet.ServletRequest.getParameter": ["GET", "POST"],
+	"javax.servlet.ServletRequest.getParameterValues": ["GET", "POST"],
+	"javax.servlet.http.HttpServletRequest.getHeader": ["HEADER"],
+	"javax.servlet.http.HttpServletRequest.getQueryString": ["GET"],
+	"javax.servlet.http.HttpServletRequest.getCookies": ["COOKIE"],
+	"io.grpc.MethodDescriptor.parseRequest": ["GRPC"],
+	"org.apache.struts2.dispatcher.multipart.MultiPartRequest.getParameterValues": ["GET", "POST"],
+	"javax.servlet.ServletRequest.getInputStream": ["GET", "POST"],
+	"javax.servlet.ServletRequest.getParameterNames()": ["GET", "POST"],
+	"javax.servlet.ServletRequest.getParameterMap": ["GET", "POST"],
+	"javax.servlet.ServletRequest.getReader": ["GET", "POST"],
+	"javax.servlet.http.HttpServletRequest.getHeaderNames()": ["HEADER"],
+	"javax.servlet.http.HttpServletRequest.getParts": ["GET", "POST"],
+	"javax.servlet.http.HttpServletRequest.getPart": ["GET", "POST"],
+	"javax.servlet.http.HttpServletRequest.getInputStream": ["GET", "POST"],
+	"javax.servlet.http.HttpServletRequest.getHeaders": ["HEADER"],
+	"javax.servlet.http.HttpServletRequest.getParameter": ["GET", "POST"],
+	"javax.servlet.http.HttpServletRequest.getParameterValues": ["GET", "POST"],
+	"javax.servlet.http.HttpServletRequest.getParameterNames()": ["GET", "POST"],
+	"javax.servlet.http.HttpServletRequest.getParameterMap()": ["GET", "POST"],
+	"javax.servlet.http.HttpServletRequest.getReader()": ["POST"],
+	"org.apache.commons.fileupload.FileUploadBase.parseRequest(org.apache.commons.fileupload.RequestContext)": ["POST"],
+	"jakarta.servlet.ServletRequest.getInputStream": ["GET", "POST"],
+	"jakarta.servlet.ServletRequest.getParameter": ["GET", "POST"],
+	"jakarta.servlet.ServletRequest.getParameterNames": ["GET", "POST"],
+	"jakarta.servlet.ServletRequest.getParameterValues": ["GET", "POST"],
+	"jakarta.servlet.ServletRequest.getReader": ["GET", "POST"],
+	"jakarta.servlet.http.HttpServletRequest.getHeader": ["HEADER"],
+	"jakarta.servlet.http.HttpServletRequest.getHeaders": ["HEADER"],
+	"jakarta.servlet.http.HttpServletRequest.getHeaderNames": ["HEADER"],
+	"jakarta.servlet.http.HttpServletRequest.getParts": ["GET", "POST"],
+	"jakarta.servlet.http.HttpServletRequest.getPart": ["GET", "POST"],
+	"jakarta.servlet.http.HttpServletRequest.getCookies": ["COOKIE"],
+	"jakarta.servlet.http.HttpServletRequest.getQueryString": ["GET"],
+	"javax.servlet.http.HttpServletRequest.getSession": ["HEADER"],
+	"org.springframework.web.servlet.mvc.method.annotation.PathVariableMethodArgumentResolver.resolveName(java.lang.String,org.springframework.core.MethodParameter,org.springframework.web.context.request.NativeWebRequest)": ["PATH"],
+	"org.springframework.web.bind.annotation.support.HandlerMethodInvoker.resolvePathVariable(java.lang.String,org.springframework.core.MethodParameter,org.springframework.web.context.request.NativeWebRequest,java.lang.Object)": ["PATH"],
+	"javax.servlet.http.Part.getSubmittedFileName()": ["POST"],
+	"javax.servlet.http.Part.getName()": ["POST"],
+	"javax.servlet.http.Part.getInputStream()": ["POST"],
+	"javax.servlet.http.Part.getHeaders(java.lang.String)": ["HEADER"],
+	"javax.servlet.http.Part.getHeaderNames()": ["HEADER"],
+	"javax.servlet.http.Part.getHeader(java.lang.String)": ["HEADER"],
+	"javax.servlet.http.Part.getContentType()": ["HEADER"],
+	"javax.servlet.http.HttpServletRequest.getRequestedSessionId()": ["HEADER"],
+	"javax.servlet.http.HttpServletRequest.getQueryString()": ["HEADER"],
+	"javax.servlet.http.HttpServletRequest.getParts()": ["POST"],
+	"javax.servlet.http.HttpServletRequest.getPart(java.lang.String)": ["POST"],
+	"javax.servlet.http.HttpServletRequest.getParameterValues(java.lang.String)": ["GET", "POST"],
+	"javax.servlet.http.HttpServletRequest.getHeaders(java.lang.String)": ["HEADER"],
+	"javax.servlet.http.HttpServletRequest.getHeader(java.lang.String)": ["HEADER"],
+	"javax.servlet.http.HttpServletRequest.getCookies()": ["COOKIE"],
+	"jakarta.servlet.http.Part.getSubmittedFileName()": ["POST"],
+	"jakarta.servlet.http.Part.getName()": ["POST"],
+	"jakarta.servlet.http.Part.getInputStream()": ["POST"],
+	"jakarta.servlet.http.Part.getHeaders(java.lang.String)": ["HEADER"],
+	"jakarta.servlet.http.Part.getHeaderNames()": ["HEADER"],
+	"jakarta.servlet.http.Part.getHeader(java.lang.String)": ["HEADER"],
+	"jakarta.servlet.http.Part.getContentType()": ["HEADER"],
+	"jakarta.servlet.http.HttpServletRequest.getRequestedSessionId()": ["HEADER"],
+	"jakarta.servlet.http.HttpServletRequest.getQueryString()": ["GET"],
+	"jakarta.servlet.http.HttpServletRequest.getParts()": ["POST"],
+	"jakarta.servlet.http.HttpServletRequest.getPart(java.lang.String)": ["POST"],
+	"jakarta.servlet.http.HttpServletRequest.getParameter(java.lang.String)": ["GET", "POST"],
+	"jakarta.servlet.http.HttpServletRequest.getHeaders(java.lang.String)": ["HEADER"],
+	"jakarta.servlet.http.HttpServletRequest.getHeaderNames()": ["HEADER"],
+	"jakarta.servlet.http.HttpServletRequest.getHeader(java.lang.String)": ["HEADER"],
+	"jakarta.servlet.http.HttpServletRequest.getCookies()": ["COOKIE"],
+	"javax.servlet.http.HttpServletRequest.getParameter(java.lang.String)": ["GET", "POST"],
+	"javax.servlet.http.HttpServletRequest.getInputStream()": ["POST"],
+	"org.springframework.http.server.ServletServerHttpRequest.getBodyFromServletRequestParameters(javax.servlet.http.HttpServletRequest)": ["POST"],
+	"org.springframework.http.server.ServletServerHttpRequest.getBodyFromServletRequestParameters(jakarta.servlet.http.HttpServletRequest)": ["POST"]
+}
\ No newline at end of file

From 6f2f02db5274836b689d2e5a7943f5d3e5dbf8a9 Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Thu, 13 Jul 2023 12:18:51 +0800
Subject: [PATCH 135/161] feat: update 1.13.0 strategy.

---
 static/data/java_full_policy.json | 43 -------------------------------
 static/data/java_policy.json      | 33 ------------------------
 2 files changed, 76 deletions(-)

diff --git a/static/data/java_full_policy.json b/static/data/java_full_policy.json
index 9973db80a..b0e1986d3 100644
--- a/static/data/java_full_policy.json
+++ b/static/data/java_full_policy.json
@@ -15200,49 +15200,6 @@
         "type": 4,
         "value": "xxe"
     },
-    {
-        "details": [
-            {
-                "command": "",
-                "created_by": 1,
-                "enable": 1,
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "language": 1,
-                "source": "",
-                "stack_blacklist": [],
-                "system_type": 1,
-                "tags": [],
-                "target": "",
-                "track": "",
-                "type": 3,
-                "untags": [],
-                "value": "javax.xml.parsers.DocumentBuilderFactory.setFeature(java.lang.String, boolean)"
-            },
-            {
-                "command": "",
-                "created_by": 1,
-                "enable": 1,
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "language": 1,
-                "source": "",
-                "stack_blacklist": [],
-                "system_type": 1,
-                "tags": [],
-                "target": "",
-                "track": "",
-                "type": 3,
-                "untags": [],
-                "value": "javax.xml.parsers.SAXParserFactory.setFeature(java.lang.String, boolean)"
-            }
-        ],
-        "enable": 1,
-        "type": 3,
-        "value": "xxe-filter"
-    },
     {
         "details": [
             {
diff --git a/static/data/java_policy.json b/static/data/java_policy.json
index 6b58a26a9..982931e9e 100644
--- a/static/data/java_policy.json
+++ b/static/data/java_policy.json
@@ -10184,39 +10184,6 @@
         "type": 4,
         "value": "xxe"
     },
-    {
-        "details": [
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "source": "",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "",
-                "untags": [],
-                "value": "javax.xml.parsers.DocumentBuilderFactory.setFeature(java.lang.String, boolean)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "source": "",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "",
-                "untags": [],
-                "value": "javax.xml.parsers.SAXParserFactory.setFeature(java.lang.String, boolean)"
-            }
-        ],
-        "enable": 1,
-        "type": 3,
-        "value": "xxe-filter"
-    },
     {
         "details": [
             {

From a8a5ae97e352de70e381ac00feb0ea3115d71a1e Mon Sep 17 00:00:00 2001
From: st1020 
Date: Thu, 13 Jul 2023 12:36:35 +0800
Subject: [PATCH 136/161] feat: change init schema log level

---
 dongtai_common/utils/init_schema.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dongtai_common/utils/init_schema.py b/dongtai_common/utils/init_schema.py
index 87d487515..823081563 100644
--- a/dongtai_common/utils/init_schema.py
+++ b/dongtai_common/utils/init_schema.py
@@ -41,6 +41,6 @@ def init_schema() -> None:
                     filepath,
                 )
         except Exception as e:
-            logger.error(
+            logger.debug(
                 f"unable to get schema: view {view} of path {path}", exc_info=e
             )

From 6b9de24bef5d1335c78edfa8f40a12c26e89f4cd Mon Sep 17 00:00:00 2001
From: tscuite 
Date: Thu, 13 Jul 2023 13:44:47 +0800
Subject: [PATCH 137/161] feat: update ci

---
 .github/workflows/deploy-dev.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/deploy-dev.yaml b/.github/workflows/deploy-dev.yaml
index 395390826..6d49b6853 100644
--- a/.github/workflows/deploy-dev.yaml
+++ b/.github/workflows/deploy-dev.yaml
@@ -83,9 +83,9 @@ jobs:
       - name: Set the value
         id: release
         run: |
-          if [ ${{ steps.version.outputs.GITHUB_REF }} = develop ] ; then echo "helm_ns=test" >> $GITHUB_ENV
-          elif [ ${{ steps.version.outputs.GITHUB_REF }} = beta ] ; then echo "helm_ns=beta" >> $GITHUB_ENV
-          else echo "helm_ns=main" >> $GITHUB_ENV ;fi
+          if [ ${{ steps.version.outputs.GITHUB_REF }} = develop ] ; then echo "helm_ns=test" >> $GITHUB_ENV; echo "helm_mysql=test" >> $GITHUB_ENV 
+          elif [ ${{ steps.version.outputs.GITHUB_REF }} = beta ] ; then echo "helm_ns=beta" >> $GITHUB_ENV; echo "helm_mysql=beta" >> $GITHUB_ENV 
+          else echo "helm_ns=main" >> $GITHUB_ENV ; echo "helm_mysql=temp" >> $GITHUB_ENV ;fi
 
       - name: deploy to cluster
         uses: wahyd4/kubectl-helm-action@master
@@ -95,5 +95,5 @@ jobs:
           args: |
             git clone https://github.com/HXSecurity/DongTai.git
             helm upgrade --install huoxian --create-namespace -n iast-${{ env.helm_ns }} ./DongTai/deploy/kubernetes/helm/ \
-            --set sca.sca_token=${{ secrets.TOKEN_SCA }} --set usb.usb_token=${{ secrets.TOKEN_SCA }} --set mysql.host=iast-mysql-test.huoxian.cn \
+            --set sca.sca_token=${{ secrets.TOKEN_SCA }} --set usb.usb_token=${{ secrets.TOKEN_SCA }} --set mysql.host=iast-mysql-${{ env.helm_mysql }}.huoxian.cn \
             --set tag=${{ steps.version.outputs.GITHUB_REF }}-latest --set build.server_number=iast${{github.run_number}} --values https://charts.dongtai.io/devops.yaml
\ No newline at end of file

From 0a388e1c0d751e704bd53369bfbca10221b1232d Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Thu, 13 Jul 2023 14:14:55 +0800
Subject: [PATCH 138/161] feat: update 1.13.0 strategy.

---
 static/data/java_full_policy.json | 115 ++++++++++--------------------
 static/data/java_policy.json      |  85 +++++++---------------
 2 files changed, 62 insertions(+), 138 deletions(-)

diff --git a/static/data/java_full_policy.json b/static/data/java_full_policy.json
index 9973db80a..9c00b6ba3 100644
--- a/static/data/java_full_policy.json
+++ b/static/data/java_full_policy.json
@@ -8930,6 +8930,42 @@
     },
     {
         "details": [
+            {
+                "command": "",
+                "created_by": 1,
+                "enable": 1,
+                "ignore_blacklist": false,
+                "ignore_internal": false,
+                "inherit": "all",
+                "language": 1,
+                "source": "P1",
+                "stack_blacklist": [],
+                "system_type": 1,
+                "tags": [],
+                "target": "R",
+                "track": "false",
+                "type": 2,
+                "untags": [],
+                "value": "org.springframework.http.server.ServletServerHttpRequest.getBodyFromServletRequestParameters(jakarta.servlet.http.HttpServletRequest)"
+            },
+            {
+                "command": "",
+                "created_by": 1,
+                "enable": 1,
+                "ignore_blacklist": false,
+                "ignore_internal": false,
+                "inherit": "all",
+                "language": 1,
+                "source": "P1",
+                "stack_blacklist": [],
+                "system_type": 1,
+                "tags": [],
+                "target": "R",
+                "track": "false",
+                "type": 2,
+                "untags": [],
+                "value": "org.springframework.http.server.ServletServerHttpRequest.getBodyFromServletRequestParameters(javax.servlet.http.HttpServletRequest)"
+            },
             {
                 "command": "",
                 "created_by": 1,
@@ -11380,42 +11416,6 @@
                 "untags": [],
                 "value": "org.springframework.cloud.config.server.resource.ResourceController.retrieve(java.lang.String,java.lang.String,java.lang.String,java.lang.String,boolean)"
             },
-            {
-                "command": "",
-                "created_by": 1,
-                "enable": 1,
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "language": 1,
-                "source": "P1",
-                "stack_blacklist": [],
-                "system_type": 1,
-                "tags": [],
-                "target": "R",
-                "track": "false",
-                "type": 1,
-                "untags": [],
-                "value": "org.springframework.http.server.ServletServerHttpRequest.getBodyFromServletRequestParameters(jakarta.servlet.http.HttpServletRequest)"
-            },
-            {
-                "command": "",
-                "created_by": 1,
-                "enable": 1,
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "language": 1,
-                "source": "P1",
-                "stack_blacklist": [],
-                "system_type": 1,
-                "tags": [],
-                "target": "R",
-                "track": "false",
-                "type": 1,
-                "untags": [],
-                "value": "org.springframework.http.server.ServletServerHttpRequest.getBodyFromServletRequestParameters(javax.servlet.http.HttpServletRequest)"
-            },
             {
                 "command": "",
                 "created_by": 1,
@@ -15200,49 +15200,6 @@
         "type": 4,
         "value": "xxe"
     },
-    {
-        "details": [
-            {
-                "command": "",
-                "created_by": 1,
-                "enable": 1,
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "language": 1,
-                "source": "",
-                "stack_blacklist": [],
-                "system_type": 1,
-                "tags": [],
-                "target": "",
-                "track": "",
-                "type": 3,
-                "untags": [],
-                "value": "javax.xml.parsers.DocumentBuilderFactory.setFeature(java.lang.String, boolean)"
-            },
-            {
-                "command": "",
-                "created_by": 1,
-                "enable": 1,
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "language": 1,
-                "source": "",
-                "stack_blacklist": [],
-                "system_type": 1,
-                "tags": [],
-                "target": "",
-                "track": "",
-                "type": 3,
-                "untags": [],
-                "value": "javax.xml.parsers.SAXParserFactory.setFeature(java.lang.String, boolean)"
-            }
-        ],
-        "enable": 1,
-        "type": 3,
-        "value": "xxe-filter"
-    },
     {
         "details": [
             {
diff --git a/static/data/java_policy.json b/static/data/java_policy.json
index 6b58a26a9..d250bf38b 100644
--- a/static/data/java_policy.json
+++ b/static/data/java_policy.json
@@ -6218,6 +6218,32 @@
     },
     {
         "details": [
+            {
+                "command": "",
+                "ignore_blacklist": false,
+                "ignore_internal": false,
+                "inherit": "all",
+                "source": "P1",
+                "stack_blacklist": [],
+                "tags": [],
+                "target": "R",
+                "track": "false",
+                "untags": [],
+                "value": "org.springframework.http.server.ServletServerHttpRequest.getBodyFromServletRequestParameters(jakarta.servlet.http.HttpServletRequest)"
+            },
+            {
+                "command": "",
+                "ignore_blacklist": false,
+                "ignore_internal": false,
+                "inherit": "all",
+                "source": "P1",
+                "stack_blacklist": [],
+                "tags": [],
+                "target": "R",
+                "track": "false",
+                "untags": [],
+                "value": "org.springframework.http.server.ServletServerHttpRequest.getBodyFromServletRequestParameters(javax.servlet.http.HttpServletRequest)"
+            },
             {
                 "command": "",
                 "ignore_blacklist": false,
@@ -7467,32 +7493,6 @@
                 "untags": [],
                 "value": "org.springframework.cloud.config.server.resource.ResourceController.retrieve(java.lang.String,java.lang.String,java.lang.String,java.lang.String,boolean)"
             },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "R",
-                "track": "false",
-                "untags": [],
-                "value": "org.springframework.http.server.ServletServerHttpRequest.getBodyFromServletRequestParameters(jakarta.servlet.http.HttpServletRequest)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "all",
-                "source": "P1",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "R",
-                "track": "false",
-                "untags": [],
-                "value": "org.springframework.http.server.ServletServerHttpRequest.getBodyFromServletRequestParameters(javax.servlet.http.HttpServletRequest)"
-            },
             {
                 "command": "",
                 "ignore_blacklist": false,
@@ -10184,39 +10184,6 @@
         "type": 4,
         "value": "xxe"
     },
-    {
-        "details": [
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "source": "",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "",
-                "untags": [],
-                "value": "javax.xml.parsers.DocumentBuilderFactory.setFeature(java.lang.String, boolean)"
-            },
-            {
-                "command": "",
-                "ignore_blacklist": false,
-                "ignore_internal": false,
-                "inherit": "true",
-                "source": "",
-                "stack_blacklist": [],
-                "tags": [],
-                "target": "",
-                "track": "",
-                "untags": [],
-                "value": "javax.xml.parsers.SAXParserFactory.setFeature(java.lang.String, boolean)"
-            }
-        ],
-        "enable": 1,
-        "type": 3,
-        "value": "xxe-filter"
-    },
     {
         "details": [
             {

From 51a9dd9ec9c355dca0c88cbb5714f7289e194be5 Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Thu, 13 Jul 2023 15:03:30 +0800
Subject: [PATCH 139/161] fix: api documents fix.

---
 dongtai_web/aggr_vul/app_vul_list.py    | 6 +++---
 dongtai_web/aggr_vul/app_vul_summary.py | 5 ++---
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/dongtai_web/aggr_vul/app_vul_list.py b/dongtai_web/aggr_vul/app_vul_list.py
index b05f5d44c..96eaac066 100644
--- a/dongtai_web/aggr_vul/app_vul_list.py
+++ b/dongtai_web/aggr_vul/app_vul_list.py
@@ -26,6 +26,7 @@
 from itertools import groupby
 from collections import defaultdict
 from dongtai_conf.patch import patch_point
+from dongtai_common.utils.const import OPERATE_GET
 
 INT_LIMIT: int = 2**64 - 1
 
@@ -34,9 +35,8 @@ class GetAppVulsList(UserEndPoint):
 
     @extend_schema_with_envcheck(
         request=AggregationArgsSerializer,
-        tags=[_('app VulList')],
-        summary=_('app List Select'),
-        description=_("select sca vul and app vul by keywords"),
+        tags=[_('Vulnerability'), OPERATE_GET],
+        summary="应用漏洞列表",
     )
     def post(self, request):
         """
diff --git a/dongtai_web/aggr_vul/app_vul_summary.py b/dongtai_web/aggr_vul/app_vul_summary.py
index 684e7796c..bad545c5a 100644
--- a/dongtai_web/aggr_vul/app_vul_summary.py
+++ b/dongtai_web/aggr_vul/app_vul_summary.py
@@ -102,9 +102,8 @@ def get_annotate_data(
 class GetAppVulsSummary(UserEndPoint):
     @extend_schema_with_envcheck(
         request=AggregationArgsSerializer,
-        tags=[_("app Vul count")],
-        summary=_("app List count"),
-        description=_("select   app vul by keywords"),
+        tags=[_('Vulnerability'), OPERATE_GET],
+        summary="应用漏洞列表统计",
     )
     def post(self, request):
         """

From c02c9b300d6656c52b86af93f8b0793892084fac Mon Sep 17 00:00:00 2001
From: st1020 
Date: Thu, 13 Jul 2023 15:23:16 +0800
Subject: [PATCH 140/161] fix: schema error

---
 dongtai_web/aggr_vul/app_vul_list.py           | 3 ++-
 dongtai_web/aggr_vul/app_vul_summary.py        | 4 +++-
 dongtai_web/dongtai_sca/views/newpackage.py    | 3 ++-
 dongtai_web/views/engine_method_pool_search.py | 3 ++-
 dongtai_web/views/profile.py                   | 3 ++-
 5 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/dongtai_web/aggr_vul/app_vul_list.py b/dongtai_web/aggr_vul/app_vul_list.py
index b05f5d44c..dc49406fe 100644
--- a/dongtai_web/aggr_vul/app_vul_list.py
+++ b/dongtai_web/aggr_vul/app_vul_list.py
@@ -26,6 +26,7 @@
 from itertools import groupby
 from collections import defaultdict
 from dongtai_conf.patch import patch_point
+from dongtai_common.utils.const import OPERATE_GET
 
 INT_LIMIT: int = 2**64 - 1
 
@@ -34,7 +35,7 @@ class GetAppVulsList(UserEndPoint):
 
     @extend_schema_with_envcheck(
         request=AggregationArgsSerializer,
-        tags=[_('app VulList')],
+        tags=[_('app VulList'), OPERATE_GET],
         summary=_('app List Select'),
         description=_("select sca vul and app vul by keywords"),
     )
diff --git a/dongtai_web/aggr_vul/app_vul_summary.py b/dongtai_web/aggr_vul/app_vul_summary.py
index 684e7796c..cf9278e2b 100644
--- a/dongtai_web/aggr_vul/app_vul_summary.py
+++ b/dongtai_web/aggr_vul/app_vul_summary.py
@@ -13,6 +13,8 @@
 from django.db.models import Q
 import logging
 from dongtai_conf.patch import patch_point
+from dongtai_common.utils.const import OPERATE_GET
+
 logger = logging.getLogger('dongtai-webapi')
 
 
@@ -102,7 +104,7 @@ def get_annotate_data(
 class GetAppVulsSummary(UserEndPoint):
     @extend_schema_with_envcheck(
         request=AggregationArgsSerializer,
-        tags=[_("app Vul count")],
+        tags=[_("app Vul count"), OPERATE_GET],
         summary=_("app List count"),
         description=_("select   app vul by keywords"),
     )
diff --git a/dongtai_web/dongtai_sca/views/newpackage.py b/dongtai_web/dongtai_sca/views/newpackage.py
index 7e723ff7d..e9bb79825 100644
--- a/dongtai_web/dongtai_sca/views/newpackage.py
+++ b/dongtai_web/dongtai_sca/views/newpackage.py
@@ -15,6 +15,7 @@
 from dongtai_common.models.assetv2 import AssetV2, AssetV2Global
 from rest_framework.serializers import ValidationError
 from dongtai_common.serializers.assetv2 import PackeageScaAssetDetailSerializer
+from dongtai_common.utils.const import OPERATE_GET
 
 logger = logging.getLogger(__name__)
 
@@ -74,7 +75,7 @@ class Meta:
 class PackageList(UserEndPoint):
 
     @extend_schema_with_envcheck_v2(request=PackageListArgsSerializer,
-                                    tags=[_('Component')],
+                                    tags=[_('Component'), OPERATE_GET],
                                     summary=_("Component List"),
                                     responses={200: _NewResponseSerializer})
     def post(self, request):
diff --git a/dongtai_web/views/engine_method_pool_search.py b/dongtai_web/views/engine_method_pool_search.py
index 099a81451..118591255 100644
--- a/dongtai_web/views/engine_method_pool_search.py
+++ b/dongtai_web/views/engine_method_pool_search.py
@@ -22,6 +22,7 @@
 from django.utils.translation import gettext_lazy
 from dongtai_conf.settings import ELASTICSEARCH_STATE
 import logging
+from dongtai_common.utils.const import OPERATE_GET
 
 logger = logging.getLogger('dongtai-webapi')
 
@@ -144,7 +145,7 @@ class MethodPoolSearchResponseSer(serializers.Serializer):
 class MethodPoolSearchProxy(AnonymousAndUserEndPoint):
     @extend_schema_with_envcheck(
         request=MethodPoolSearchProxySer,
-        tags=[_('Method Pool')],
+        tags=[_('Method Pool'), OPERATE_GET],
         summary=_('Method Pool Search'),
         description=_("Search for the method pool information according to the following conditions, the default is regular expression input, regular specifications refer to REGEX POSIX 1003.2"
                       ),
diff --git a/dongtai_web/views/profile.py b/dongtai_web/views/profile.py
index 157cd93bb..f52945ae1 100644
--- a/dongtai_web/views/profile.py
+++ b/dongtai_web/views/profile.py
@@ -8,6 +8,7 @@
 from rest_framework import serializers
 from rest_framework.serializers import ValidationError
 from django.forms.models import model_to_dict
+from dongtai_common.utils.const import OPERATE_GET
 
 
 class ProfilepostArgsSer(serializers.Serializer):
@@ -65,7 +66,7 @@ class ProfileBatchGetEndpoint(UserEndPoint):
         request=ProfileBatchGetArgsSer,
         description=_("Get Profile with key batch"),
         response_schema=ProfileBatchGetResSer(many=True),
-        tags=[_('Profile')])
+        tags=[_('Profile'), OPERATE_GET])
     def post(self, request):
         keys = request.data.get('keys', None)
         profiles = IastProfile.objects.filter(key__in=keys).all()

From 9fd8c65b9271578cf6d27df98e3486f60d0a86ac Mon Sep 17 00:00:00 2001
From: st1020 
Date: Thu, 13 Jul 2023 18:27:08 +0800
Subject: [PATCH 141/161] fix: add miss api vul_recheck_payload

---
 dongtai_web/urls.py                           |  17 ++
 dongtai_web/vul_recheck_payload/__init__.py   |   0
 .../vul_recheck_payload.py                    | 147 ++++++++++++++++++
 3 files changed, 164 insertions(+)
 create mode 100644 dongtai_web/vul_recheck_payload/__init__.py
 create mode 100644 dongtai_web/vul_recheck_payload/vul_recheck_payload.py

diff --git a/dongtai_web/urls.py b/dongtai_web/urls.py
index 7b4b7868f..16207737b 100644
--- a/dongtai_web/urls.py
+++ b/dongtai_web/urls.py
@@ -122,6 +122,7 @@
 from dongtai_web.aggregation.aggregation_project_del import DelVulProjectLevel
 
 from dongtai_web.vul_log.vul_log_view import VulLogViewSet
+from dongtai_web.vul_recheck_payload.vul_recheck_payload import VulReCheckPayloadViewSet
 from dongtai_web.header_vul.base import HeaderVulViewSet
 from dongtai_web.views.new_project_query import NewProjectVersionList
 from dongtai_web.enum.hook_rules import HookRuleEnumEndPoint
@@ -257,6 +258,22 @@
     path('vul_list_delete', DelVulMany.as_view()),
     path('project_vul_delete', DelVulProjectLevel.as_view()),
     path("vullog/", VulLogViewSet.as_view({"get": "list"})),
+    path(
+        'vul_recheck_payload/',
+        VulReCheckPayloadViewSet.as_view({
+            'get': "retrieve",
+            'put': 'update',
+            'delete': 'delete'
+        })),
+    path('vul_recheck_payload',
+         VulReCheckPayloadViewSet.as_view({
+             'get': "list",
+             'post': "create",
+         })),
+    path('vul_recheck_payload/status',
+         VulReCheckPayloadViewSet.as_view({
+             'put': "status_change",
+         })),
     path('header_vul', HeaderVulViewSet.as_view({
         'get': "list",
     })),
diff --git a/dongtai_web/vul_recheck_payload/__init__.py b/dongtai_web/vul_recheck_payload/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/dongtai_web/vul_recheck_payload/vul_recheck_payload.py b/dongtai_web/vul_recheck_payload/vul_recheck_payload.py
new file mode 100644
index 000000000..a441b86cd
--- /dev/null
+++ b/dongtai_web/vul_recheck_payload/vul_recheck_payload.py
@@ -0,0 +1,147 @@
+import time
+
+from dongtai_common.endpoint import UserEndPoint, R, TalentAdminEndPoint
+from dongtai_common.models.agent_config import IastAgentConfig
+from django.utils.translation import gettext_lazy as _
+from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
+from dongtai_web.serializers.agent_config import AgentConfigSettingSerializer
+from rest_framework.serializers import ValidationError
+from rest_framework import viewsets
+from dongtai_common.models.vul_recheck_payload import IastVulRecheckPayload
+from rest_framework import serializers
+from django.db.models import Q
+from django.forms.models import model_to_dict
+
+
+def get_or_none(classmodel, function, **kwargs):
+    try:
+        if function:
+            return function(classmodel.objects).get(**kwargs)
+        return classmodel.objects.get(**kwargs)
+    except classmodel.DoesNotExist:
+        return None
+
+
+class IastVulRecheckPayloadSerializer(serializers.Serializer):
+    value = serializers.CharField()
+    status = serializers.IntegerField()
+    strategy_id = serializers.IntegerField()
+    language_id = serializers.IntegerField()
+
+
+class IastVulRecheckPayloadListSerializer(serializers.Serializer):
+    keyword = serializers.CharField(required=False, default=None)
+    page = serializers.IntegerField()
+    page_size = serializers.IntegerField()
+    strategy_id = serializers.IntegerField(required=False, default=None)
+    language_id = serializers.IntegerField(required=False, default=None)
+
+
+def vul_recheck_payload_create(data, user_id):
+    IastVulRecheckPayload.objects.create(strategy_id=data['strategy_id'],
+                                         user_id=user_id,
+                                         value=data['value'],
+                                         status=data['status'],
+                                         language_id=data['language_id'])
+
+
+class AgentConfigSettingBatchV2Serializer(serializers.Serializer):
+    ids = serializers.ListField(child=serializers.IntegerField())
+
+
+def payload_update(pk, data):
+    IastVulRecheckPayload.objects.filter(pk=pk).update(**data)
+
+
+class VulReCheckPayloadViewSet(UserEndPoint, viewsets.ViewSet):
+    name = "api-v1-vul-recheck-payload"
+    description = _("config recheck payload V2")
+
+    def create(self, request):
+        ser = IastVulRecheckPayloadSerializer(data=request.data)
+        try:
+            if ser.is_valid(True):
+                pass
+        except ValidationError as e:
+            return R.failure(data=e.detail)
+        vul_recheck_payload_create(ser.data, request.user.id)
+        return R.success()
+
+    def retrieve(self, request, pk):
+        obj = get_or_none(
+            IastVulRecheckPayload,
+            lambda x: x.values('id', 'user__username', 'strategy__vul_name',
+                               'value', 'user_id', 'strategy_id', 'status',
+                               'create_time', 'language_id'),
+            pk=pk, user_id=request.user.id)
+        if obj is None:
+            return R.failure()
+        return R.success(data=obj)
+
+    def list(self, request):
+        ser = IastVulRecheckPayloadListSerializer(data=request.query_params)
+        try:
+            if ser.is_valid(True):
+                pass
+        except ValidationError as e:
+            return R.failure(data=e.detail)
+        q = Q(user_id=request.user.id) & ~Q(status=-1)
+        keyword = ser.data['keyword']
+        strategy_id = ser.data['strategy_id']
+        language_id = ser.data['language_id']
+        page = ser.data['page']
+        page_size = ser.data['page_size']
+        if keyword:
+            q = q & Q(value__icontains=keyword)
+        if strategy_id:
+            q = q & Q(strategy_id=strategy_id)
+        if language_id:
+            q = q & Q(language_id=language_id)
+        queryset = IastVulRecheckPayload.objects.filter(q).order_by(
+            '-create_time').values('id', 'user__username',
+                                   'strategy__vul_name', 'value', 'user_id',
+                                   'strategy_id', 'status', 'create_time',
+                                   'language_id')
+        page_summary, page_data = self.get_paginator(queryset, page, page_size)
+        return R.success(page=page_summary,
+                         data=list(page_data))
+
+    def update(self, request, pk):
+        ser = IastVulRecheckPayloadSerializer(data=request.data)
+        try:
+            if ser.is_valid(True):
+                pass
+        except ValidationError as e:
+            return R.failure(data=e.detail)
+        if IastVulRecheckPayload.objects.filter(
+                pk=pk, user_id=request.user.id).exists():
+            payload_update(pk, ser.data)
+            return R.success()
+        return R.success()
+
+    def delete(self, request, pk):
+        if IastVulRecheckPayload.objects.filter(
+                pk=pk, user_id=request.user.id).exists():
+            IastVulRecheckPayload.objects.filter(pk=pk).update(status=-1)
+            return R.success()
+        return R.failure()
+
+    def status_change(self, request):
+        mode = request.data.get('mode', 1)
+        q = ~Q(status=-1) & Q(user_id=request.user.id)
+        if mode == 1:
+            ids = request.data.get('ids', [])
+            status = request.data.get('status', 0)
+            q = q & Q(pk__in=ids)
+            IastVulRecheckPayload.objects.filter(q).update(status=status)
+        elif mode == 2:
+            status = request.data.get('status', 0)
+            q = q
+            IastVulRecheckPayload.objects.filter(q).update(status=status)
+        return R.success()
+
+    def status_all(self, request):
+        status = request.data.get('status', 0)
+        q = ~Q(status=-1)
+        IastVulRecheckPayload.objects.filter(q).update(status=status)
+        return R.success()

From 964cf2655814701c24c0577277696d20e7d9ffdb Mon Sep 17 00:00:00 2001
From: st1020 
Date: Thu, 13 Jul 2023 18:32:03 +0800
Subject: [PATCH 142/161] fix: shema error

---
 dongtai_web/aggr_vul/app_vul_summary.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dongtai_web/aggr_vul/app_vul_summary.py b/dongtai_web/aggr_vul/app_vul_summary.py
index bad545c5a..fc9c389dd 100644
--- a/dongtai_web/aggr_vul/app_vul_summary.py
+++ b/dongtai_web/aggr_vul/app_vul_summary.py
@@ -13,6 +13,7 @@
 from django.db.models import Q
 import logging
 from dongtai_conf.patch import patch_point
+from dongtai_common.utils.const import OPERATE_GET
 logger = logging.getLogger('dongtai-webapi')
 
 

From 07c38843c686a0ecce5fabb668ae9983ead81d2c Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Thu, 13 Jul 2023 19:09:31 +0800
Subject: [PATCH 143/161] fix: new graph check node exists.

---
 dongtai_common/engine/vul_engine.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/dongtai_common/engine/vul_engine.py b/dongtai_common/engine/vul_engine.py
index 85df6541b..ffd039372 100644
--- a/dongtai_common/engine/vul_engine.py
+++ b/dongtai_common/engine/vul_engine.py
@@ -169,6 +169,8 @@ def search(self, method_pool, vul_method_signature, vul_type=None):
         # Checkout each pair source/target have a path or not
         # It may lost sth when muliti paths exists.
         for s, t in product(source_methods, vul_methods):
+            if not g.hasNode(s) or not g.hasNode(t):
+                continue
             dij_obj = nk.distance.BidirectionalDijkstra(g, s, t).run()
             if dij_obj.getDistance() != 1.7976931348623157e+308:  # INF here!
                 logger.info('find sink here!')

From 0c8b09941caeb1b07fe7fbd3ecb4f19103d10b32 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 15 Jul 2023 00:37:16 +0000
Subject: [PATCH 144/161] build(deps): bump cryptography from 41.0.0 to 41.0.2

Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.0 to 41.0.2.
- [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/pyca/cryptography/compare/41.0.0...41.0.2)

---
updated-dependencies:
- dependency-name: cryptography
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
---
 Pipfile      |   2 +-
 Pipfile.lock | 296 ++++++++++++++++++++++++++-------------------------
 2 files changed, 151 insertions(+), 147 deletions(-)

diff --git a/Pipfile b/Pipfile
index 55486ba7b..7814f0655 100644
--- a/Pipfile
+++ b/Pipfile
@@ -69,7 +69,7 @@ botocore = "==1.27.91"
 setuptools = "==65.5.1"
 elasticsearch = "==7.17.7"
 django-mock-queries = "==v2.1.7"
-cryptography = "==41.0.0"
+cryptography = "==41.0.2"
 jsonschema = "==4.17.0"
 pillow = "==9.3.0"
 pyrsistent = "==0.19.1"
diff --git a/Pipfile.lock b/Pipfile.lock
index ee1466023..1da8e7213 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "bd16237109411b522dc3f42ae2bcf1c44ddb2dec4f74aee4ef66f17e55794fa4"
+            "sha256": "c71bdb338f44bba9038b8f6f777a13c6d11564091a415704869e7eee0f64e06a"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -303,23 +303,23 @@
                 "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac",
                 "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==3.2.0"
         },
         "click": {
             "hashes": [
-                "sha256:2739815aaa5d2c986a88f1e9230c55e17f0caad3d958a5e13ad0797c166db9e3",
-                "sha256:b97d0c74955da062a7d4ef92fadb583806a585b2ea81958a81bd72726cbb8e37"
+                "sha256:4be4b1af8d665c6d942909916d31a213a106800c47d0eeba73d34da3cbc11367",
+                "sha256:e576aa487d679441d7d30abb87e1b43d24fc53bffb8758443b1a9e1cee504548"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==8.1.4"
+            "version": "==8.1.5"
         },
         "click-didyoumean": {
             "hashes": [
                 "sha256:a0713dc7a1de3f06bc0df5a9567ad19ead2d3d5689b434768a6145bff77c0667",
                 "sha256:f184f0d851d96b6d29297354ed981b7dd71df7ff500d82fa6d11f0856bee8035"
             ],
-            "markers": "python_full_version >= '3.6.2' and python_full_version < '4.0.0'",
+            "markers": "python_version < '4' and python_full_version >= '3.6.2'",
             "version": "==0.3.0"
         },
         "click-plugins": {
@@ -348,28 +348,32 @@
         },
         "cryptography": {
             "hashes": [
-                "sha256:0ddaee209d1cf1f180f1efa338a68c4621154de0afaef92b89486f5f96047c55",
-                "sha256:14754bcdae909d66ff24b7b5f166d69340ccc6cb15731670435efd5719294895",
-                "sha256:344c6de9f8bda3c425b3a41b319522ba3208551b70c2ae00099c205f0d9fd3be",
-                "sha256:34d405ea69a8b34566ba3dfb0521379b210ea5d560fafedf9f800a9a94a41928",
-                "sha256:3680248309d340fda9611498a5319b0193a8dbdb73586a1acf8109d06f25b92d",
-                "sha256:3c5ef25d060c80d6d9f7f9892e1d41bb1c79b78ce74805b8cb4aa373cb7d5ec8",
-                "sha256:4ab14d567f7bbe7f1cdff1c53d5324ed4d3fc8bd17c481b395db224fb405c237",
-                "sha256:5c1f7293c31ebc72163a9a0df246f890d65f66b4a40d9ec80081969ba8c78cc9",
-                "sha256:6b71f64beeea341c9b4f963b48ee3b62d62d57ba93eb120e1196b31dc1025e78",
-                "sha256:7d92f0248d38faa411d17f4107fc0bce0c42cae0b0ba5415505df72d751bf62d",
-                "sha256:8362565b3835ceacf4dc8f3b56471a2289cf51ac80946f9087e66dc283a810e0",
-                "sha256:84a165379cb9d411d58ed739e4af3396e544eac190805a54ba2e0322feb55c46",
-                "sha256:88ff107f211ea696455ea8d911389f6d2b276aabf3231bf72c8853d22db755c5",
-                "sha256:9f65e842cb02550fac96536edb1d17f24c0a338fd84eaf582be25926e993dde4",
-                "sha256:a4fc68d1c5b951cfb72dfd54702afdbbf0fb7acdc9b7dc4301bbf2225a27714d",
-                "sha256:b7f2f5c525a642cecad24ee8670443ba27ac1fab81bba4cc24c7b6b41f2d0c75",
-                "sha256:b846d59a8d5a9ba87e2c3d757ca019fa576793e8758174d3868aecb88d6fc8eb",
-                "sha256:bf8fc66012ca857d62f6a347007e166ed59c0bc150cefa49f28376ebe7d992a2",
-                "sha256:f5d0bf9b252f30a31664b6f64432b4730bb7038339bd18b1fafe129cfc2be9be"
-            ],
-            "index": "pypi",
-            "version": "==41.0.0"
+                "sha256:01f1d9e537f9a15b037d5d9ee442b8c22e3ae11ce65ea1f3316a41c78756b711",
+                "sha256:079347de771f9282fbfe0e0236c716686950c19dee1b76240ab09ce1624d76d7",
+                "sha256:182be4171f9332b6741ee818ec27daff9fb00349f706629f5cbf417bd50e66fd",
+                "sha256:192255f539d7a89f2102d07d7375b1e0a81f7478925b3bc2e0549ebf739dae0e",
+                "sha256:2a034bf7d9ca894720f2ec1d8b7b5832d7e363571828037f9e0c4f18c1b58a58",
+                "sha256:342f3767e25876751e14f8459ad85e77e660537ca0a066e10e75df9c9e9099f0",
+                "sha256:439c3cc4c0d42fa999b83ded80a9a1fb54d53c58d6e59234cfe97f241e6c781d",
+                "sha256:49c3222bb8f8e800aead2e376cbef687bc9e3cb9b58b29a261210456a7783d83",
+                "sha256:674b669d5daa64206c38e507808aae49904c988fa0a71c935e7006a3e1e83831",
+                "sha256:7a9a3bced53b7f09da251685224d6a260c3cb291768f54954e28f03ef14e3766",
+                "sha256:7af244b012711a26196450d34f483357e42aeddb04128885d95a69bd8b14b69b",
+                "sha256:7d230bf856164de164ecb615ccc14c7fc6de6906ddd5b491f3af90d3514c925c",
+                "sha256:84609ade00a6ec59a89729e87a503c6e36af98ddcd566d5f3be52e29ba993182",
+                "sha256:9a6673c1828db6270b76b22cc696f40cde9043eb90373da5c2f8f2158957f42f",
+                "sha256:9b6d717393dbae53d4e52684ef4f022444fc1cce3c48c38cb74fca29e1f08eaa",
+                "sha256:9c3fe6534d59d071ee82081ca3d71eed3210f76ebd0361798c74abc2bcf347d4",
+                "sha256:a719399b99377b218dac6cf547b6ec54e6ef20207b6165126a280b0ce97e0d2a",
+                "sha256:b332cba64d99a70c1e0836902720887fb4529ea49ea7f5462cf6640e095e11d2",
+                "sha256:d124682c7a23c9764e54ca9ab5b308b14b18eba02722b8659fb238546de83a76",
+                "sha256:d73f419a56d74fef257955f51b18d046f3506270a5fd2ac5febbfa259d6c0fa5",
+                "sha256:f0dc40e6f7aa37af01aba07277d3d64d5a03dc66d682097541ec4da03cc140ee",
+                "sha256:f14ad275364c8b4e525d018f6716537ae7b6d369c094805cae45300847e0894f",
+                "sha256:f772610fe364372de33d76edcd313636a25684edb94cee53fd790195f5989d14"
+            ],
+            "index": "pypi",
+            "version": "==41.0.2"
         },
         "dataclasses-json": {
             "hashes": [
@@ -1163,18 +1167,18 @@
         },
         "mock": {
             "hashes": [
-                "sha256:06f18d7d65b44428202b145a9a36e99c2ee00d1eb992df0caf881d4664377891",
-                "sha256:0e0bc5ba78b8db3667ad636d964eb963dc97a59f04c6f6214c5f0e4a8f726c56"
+                "sha256:18c694e5ae8a208cdb3d2c20a993ca1a7b0efa258c247a1e565150f477f83744",
+                "sha256:5e96aad5ccda4718e0a229ed94b2024df75cc2d55575ba5762d31f5767b8767d"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==5.0.2"
+            "version": "==5.1.0"
         },
         "model-bakery": {
             "hashes": [
                 "sha256:70e991dd503e3f1956632c621946bb8e71012172db740bab75cd9d2a447283a3",
                 "sha256:bfa5d118f91f08ef4694d4aaab168c98efc91dab086aee587e71d66a7001701a"
             ],
-            "markers": "python_version >= '3'",
+            "markers": "python_version >= '3.7'",
             "version": "==1.12.0"
         },
         "mypy": {
@@ -1401,7 +1405,7 @@
                 "sha256:04505ade687dc26dc4284b1ad19a83be2f2afe83e7a828ace0c72f3a1df72aac",
                 "sha256:9dffbe1d8acf91e3de75f3b544e4842382fc06c6babe903ac9acb74dc6e08d88"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==3.0.39"
         },
         "pycodestyle": {
@@ -1492,118 +1496,118 @@
         },
         "pydantic": {
             "hashes": [
-                "sha256:b802f5245b8576315fe619e5989fd083448fa1258638ef9dac301ca60878396d",
-                "sha256:f5581e0c79b2ec2fa25a9d30d766629811cdda022107fa73d022ab5578873ae3"
+                "sha256:614eb3321eb600c81899a88fa9858b008e3c79e0d4f1b49ab1f516b4b0c27cfb",
+                "sha256:94f13e0dcf139a5125e88283fc999788d894e14ed90cf478bcc2ee50bd4fc630"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.0.2"
+            "version": "==2.0.3"
         },
         "pydantic-core": {
             "hashes": [
-                "sha256:017700236ea2e7afbef5d3803559c80bd8720306778ebd49268de7ce9972e83e",
-                "sha256:047e782b9918f35ef534ced36f1fd2064f5581229b7a15e4d3177387a6b53134",
-                "sha256:0681472245ef182554208a25d16884c84f1c5a69f14e6169b88932e5da739a1c",
-                "sha256:06ae67547251135a1b3f8dd465797b13146295a3866bc12ddd73f7512787bb7c",
-                "sha256:080a7af828388284a68ad7d3d3eac3bcfff6a580292849aff087e7d556ec42d4",
-                "sha256:0855cf8b760fb40f97f0226cb527c8a94a2ab9d8179628beae20d6939aaeacb0",
-                "sha256:087ddbb754575618a8832ee4ab52fe7eb332f502e2a56088b53dbeb5c4efdf9f",
-                "sha256:0b5d37aedea5963f2097bddbcdb255483191646a52d40d8bb66d61c190fcac91",
-                "sha256:0be2e2812a43205728a06c9d0fd090432cd76a9bb5bff2bfcfdf8b0e27d51851",
-                "sha256:0e5761ce986ec709897b1b965fad9743f301500434bea3cbab2b6e662571580f",
-                "sha256:15eb4cb543ed36f6a4f16e3bee7aa7ed1c3757be95a3f3bbb2b82b9887131e0f",
-                "sha256:1635a37137fafbc6ee0a8c879857e05b30b1aabaa927e653872b71f1501b1502",
-                "sha256:1a5c4475510d1a9cc1458a26cfc21442223e52ce9adb640775c38739315d03c7",
-                "sha256:1c917f7a41d9d09b8b024a5d65cf37e5588ccdb6e610d2df565fb7186b1f3b1c",
-                "sha256:2278ca0b0dfbcfb1e12fa58570916dc260dc72bee5e6e342debf5329d8204688",
-                "sha256:24c3c9180a2d19d640bacc2d00f497a9a1f2abadb2a9ee201b56bb03bc5343bd",
-                "sha256:2575664f0a559a7b951a518f6f34c23cab7190f34f8220b8c8218c4f403147ee",
-                "sha256:2ca2d2d5ab65fb40dd05259965006edcc62a9d9b30102737c0a6f45bcbd254e8",
-                "sha256:2ee3ae58f271851362f6c9b33e4c9f9e866557ec7d8c03dc091e9b5aa5566cec",
-                "sha256:3747a4178139ebf3f19541285b2eb7c886890ca4eb7eec851578c02a13cc1385",
-                "sha256:4663293a36a851a860b1299c50837914269fca127434911297dd39fea9667a01",
-                "sha256:46cd323371aa7e4053010ccdb94063a4273aa9e5dbe97f8a1147faa769de8d8d",
-                "sha256:4938b32c09dbcecbeb652327cb4a449b1ef1a1bf6c8fc2c8241aa6b8f6d63b54",
-                "sha256:4ac140d54da366672f6b91f9a1e8e2d4e7e72720143353501ae886d3fca03272",
-                "sha256:4aff436c23c68449601b3fba7075b4f37ef8fbb893c8c1ed3ef898f090332b1e",
-                "sha256:4e67f9b9dfda2e42b39459cbf99d319ccb90da151e35cead3521975b2afbf673",
-                "sha256:5056afea59651c4e47ec6dadbb77ccae4742c059a3d12bc1c0e393d189d2970d",
-                "sha256:51968887d6bd1eaa7fc7759701ea8ccb470c04654beaa8ede6835b0533f206a9",
-                "sha256:5948af62f323252d56acaec8ebfca5f15933f6b72f8dbe3bf21ee97b2d10e3f0",
-                "sha256:5a014ee88980013d192a718cbb88e8cea20acd3afad69bc6d15672d05a49cdb6",
-                "sha256:60b7239206a2f61ad89c7518adfacb3ccd6662eaa07c5e437317aea2615a1f18",
-                "sha256:682ff9228c838018c47dfa89b3d84cca45f88cacde28807ab8296ec221862af4",
-                "sha256:68a2a767953c707d9575dcf14d8edee7930527ee0141a8bb612c22d1f1059f9a",
-                "sha256:6bf00f56a4468f5b03dadb672a5f1d24aea303d4ccffe8a0f548c9e36017edd3",
-                "sha256:6e3bcb4a9bc209a61ea2aceb7433ce2ece32c7e670b0c06848bf870c9b3e7d87",
-                "sha256:7345b1741bf66a9d8ed0ec291c3eabd534444e139e1ea6db5742ac9fd3be2530",
-                "sha256:74a33aa69d476773230396396afb8e11908f8dafdcfd422e746770599a3f889d",
-                "sha256:7648e48ba263ca0a8a2dc55a60a219c9133fb101ba52c89a14a29fb3d4322ca3",
-                "sha256:7684b5fb906b37e940c5df3f57118f32e033af5e4770e5ae2ae56fbd2fe1a30a",
-                "sha256:76c9c55462740d728b344e3a087775846516c3fee31ec56e2075faa7cfcafcbf",
-                "sha256:7c7ad8958aadfbcd664078002246796ecd5566b64b22f6af4fd1bbcec6bf8f60",
-                "sha256:7ff6bfe63f447a509ed4d368a7f4ba6a7abc03bc4744fc3fb30f2ffab73f3821",
-                "sha256:804cf8f6a859620f8eb754c02f7770f61c3e9c519f8338c331d555b3d6976e3c",
-                "sha256:8125152b03dd91deca5afe5b933a1994b39405adf6be2fe8dce3632319283f85",
-                "sha256:817681d111cb65f07d46496eafec815f48e1aff37713b73135a0a9eb4d3610ab",
-                "sha256:818f5cb1b209ab1295087c45717178f4bbbd2bd7eda421f7a119e7b9b736a3cb",
-                "sha256:82e09f27edab289187dd924d4d93f2a35f21aa969699b2504aa643da7fbfeff9",
-                "sha256:840238c845b0f80777151fef0003088ab91c6f7b3467edaff4932b425c4e3c3f",
-                "sha256:87cff210af3258ca0c829e3ebc849d7981bfde23a99d6cb7a3c17a163b3dbad2",
-                "sha256:8884a1dbfc5cb8c54b48446ca916d4577c1f4d901126091e4ab25d00194e065f",
-                "sha256:88a56f0f6d020b4d17641f4b4d1f9540a536d4146768d059c430e97bdb485fc1",
-                "sha256:8b9a5fc4058d64c9c826684dcdb43891c1b474a4a88dcf8dfc3e1fb5889496f8",
-                "sha256:8c0213891898fa5b404cf3edf4797e3ac7819a0708ea5473fc6432a2aa27c189",
-                "sha256:8e6ce261ccb9a986953c4dce070327e4954f9dd4cd214746dfc70efbc713b6a1",
-                "sha256:8eb4e2b71562375609c66a79f89acd4fe95c5cba23473d04952c8b14b6f908f5",
-                "sha256:90b06bb47e60173d24c7cb79670aa8dd6081797290353b9d3c66d3a23e88eb34",
-                "sha256:94d368af9e6563de6e7170a74710a2cbace7a1e9c8e507d9e3ac34c7065d7ae3",
-                "sha256:9a5fba9168fc27805553760fa8198db46eef83bf52b4e87ebbe1333b823d0e70",
-                "sha256:9b9f8bf1d7008a58fbb6eb334dc6e2f2905400cced8dadb46c4ca28f005a8562",
-                "sha256:a4ae46769d9a7138d58cd190441cac14ce954010a0081f28462ed916c8e55a4f",
-                "sha256:a772c652603855d7180015849d483a1f539351a263bb9b81bfe85193a33ce124",
-                "sha256:a8b9c2cc4c5f8169b943d24be4bd1548fe81c016d704126e3a3124a2fc164885",
-                "sha256:aa39499625239da4ec960cf4fc66b023929b24cc77fb8520289cfdb3c1986428",
-                "sha256:aa54902fa51f7d921ba80923cf1c7ff3dce796a7903300bd8824deb90e357744",
-                "sha256:ac462a28218ea7d592c7ad51b517558f4ac6565a4e53db7a4811eeaf9c9660b0",
-                "sha256:af832edd384755826e494ffdcf1fdda86e4babc42a0b18d342943fb18181040e",
-                "sha256:b1fad38db1744d27061df516e59c5025b09b0a50a337c04e6eebdbddc18951bc",
-                "sha256:b4038869ba1d8fa33863b4b1286ab07e6075a641ae269b865f94d7e10b3e800e",
-                "sha256:b4673d1f29487608d613ebcc5caa99ba15eb58450a7449fb6d800f29d90bebc1",
-                "sha256:b4815720c266e832b20e27a7a5f3772bb09fdedb31a9a34bab7b49d98967ef5a",
-                "sha256:b59a64c367f350873c40a126ffe9184d903d2126c701380b4b55753484df5948",
-                "sha256:b74906e01c7fc938ac889588ef438de812989817095c3c4904721f647d64a4d1",
-                "sha256:b815a769b019dd96be6571096f246b74f63330547e9b30244c51b4a2eb0277fc",
-                "sha256:bad7029fb2251c1ac7d3acdd607e540d40d137a7d43a5e5acdcfdbd38db3fc0a",
-                "sha256:bb471ea8650796060afc99909d9b75da583d317e52f660faf64c45f70b3bf1e2",
-                "sha256:bd95d223de5162811a7b36c73d48eac4fee03b075132f3a1b73c132ce157a60c",
-                "sha256:be3419204952bbe9b72b90008977379c52f99ae1c6e640488de4be783c345d71",
-                "sha256:c0f481aaf0119f77b200e5a5e2799b3e14c015a317eaa948f42263908735cc9f",
-                "sha256:c2d00a96fdf26295c6f25eaf9e4a233f353146a73713cd97a5f5dc6090c3aef2",
-                "sha256:c720e55cef609d50418bdfdfb5c44a76efc020ae7455505788d0113c54c7df55",
-                "sha256:cb854ec52e6e2e05b83d647695f4d913452fdd45a3dfa8233d7dab5967b3908f",
-                "sha256:cbba32fb14e199d0493c6b9c44870dab0a9c37af9f0f729068459d1849279ffd",
-                "sha256:cd62f73830d4715bc643ae39de0bd4fb9c81d6d743530074da91e77a2cccfe67",
-                "sha256:cf92dccca8f66e987f6c4378700447f82b79e86407912ab1ee06b16b82f05120",
-                "sha256:d281a10837d98db997c0247f45d138522c91ce30cf3ae7a6afdb5e709707d360",
-                "sha256:d2c790f0d928b672484eac4f5696dd0b78f3d6d148a641ea196eb49c0875e30a",
-                "sha256:d35d634d9d1ed280c87bc2a7a6217b8787eedc86f368fc2fa1c0c8c78f7d3c93",
-                "sha256:db4564aea8b3cb6cf1e5f3fd80f1ced73a255d492396d1bd8abd688795b34d63",
-                "sha256:dc737506b4a0ba2922a2626fc6d620ce50a46aebd0fe2fbcad1b93bbdd8c7e78",
-                "sha256:e17056390068afd4583d88dcf4d4495764e4e2c7d756464468e0d21abcb8931e",
-                "sha256:e68a404fad8493989d6f07b7b9e066f1d2524d7cb64db2d4e9a84c920032c67f",
-                "sha256:e7fd334b40c5e13a97becfcaba314de0dcc6f7fe21ec8f992139bcc64700e9dc",
-                "sha256:eb4301f009a44bb5db5edfe4e51a8175a4112b566baec07f4af8b1f8cb4649a2",
-                "sha256:ebf583f4d9b52abd15cc59e5f6eeca7e3e9741c6ea62d8711c00ac3acb067875",
-                "sha256:ef71e73a81a4cd7e87c93e8ff0170140fd93ba33b0f61e83da3f55f6e0a84fb4",
-                "sha256:f0eb54b11cd4fe0c6404611eef77086ade03eb1457e92910bbb4f3479efa3f79",
-                "sha256:f2de65752fff248319bcd3b29da24e205fa505607539fcd4acc4037355175b63",
-                "sha256:f5de2d4167fd4bc5ad205fb7297e25867b8e335ca08d64ed7a561d2955a2c32d",
-                "sha256:f7bcdf70c8b6e70be11c78d3c00b80a24cccfb408128f23e91ec3019bed1ecc1",
-                "sha256:fa38a76e832743866aed6b715869757074b06357d1a260163ec26d84974245fe",
-                "sha256:fc909f62325a631e1401dd07dfc386986dbcac15f98c9ff2145d930678a9d25a"
+                "sha256:019c5c41941438570dfc7d3f0ae389b2425add1775a357ce1e83ed1434f943d6",
+                "sha256:01f56d5ee70b1d39c0fd08372cc5142274070ab7181d17c86035f130eebc05b8",
+                "sha256:055f7ea6b1fbb37880d66d70eefd22dd319b09c79d2cb99b1dbfeb34b653b0b2",
+                "sha256:05b4bf8c58409586a7a04c858a86ab10f28c6c1a7c33da65e0326c59d5b0ab16",
+                "sha256:06884c07956526ac9ebfef40fe21a11605569b8fc0e2054a375fb39c978bf48f",
+                "sha256:06f33f695527f5a86e090f208978f9fd252c9cfc7e869d3b679bd71f7cb2c1fa",
+                "sha256:0aa429578e23885b3984c49d687cd05ab06f0b908ea1711a8bf7e503b7f97160",
+                "sha256:0b3d781c71b8bfb621ef23b9c874933e2cd33237c1a65cc20eeb37437f8e7e18",
+                "sha256:0dc5f516b24d24bc9e8dd9305460899f38302b3c4f9752663b396ef9848557bf",
+                "sha256:0fc7e0b056b66cc536e97ef60f48b3b289f6b3b62ac225afd4b22a42434617bf",
+                "sha256:12be3b5f54f8111ca38e6b7277f26c23ba5cb3344fae06f879a0a93dfc8b479e",
+                "sha256:1624baa76d1740711b2048f302ae9a6d73d277c55a8c3e88b53b773ebf73a971",
+                "sha256:1aefebb506bc1fe355d91d25f12bcdea7f4d7c2d9f0f6716dd025543777c99a5",
+                "sha256:1bcfb7be905aa849bd882262e1df3f75b564e2f708b4b4c7ad2d3deaf5410562",
+                "sha256:1c119e9227487ad3d7c3c737d896afe548a6be554091f9745da1f4b489c40561",
+                "sha256:20d710c1f79af930b8891bcebd84096798e4387ab64023ef41521d58f21277d3",
+                "sha256:2183a9e18cdc0de53bdaa1675f237259162abeb62d6ac9e527c359c1074dc55d",
+                "sha256:27babb9879bf2c45ed655d02639f4c30e2b9ef1b71ce59c2305bbf7287910a18",
+                "sha256:27c1bbfb9d84a75cf33b7f19b53c29eb7ead99b235fce52aced5507174ab8f98",
+                "sha256:2b79f3681481f4424d7845cc7a261d5a4baa810d656b631fa844dc9967b36a7b",
+                "sha256:2f10aa5452b865818dd0137f568d443f5e93b60a27080a01aa4b7512c7ba13a3",
+                "sha256:309f45d4d7481d6f09cb9e35c72caa0e50add4a30bb08c04c5fe5956a0158633",
+                "sha256:31acc37288b8e69e4849f618c3d5cf13b58077c1a1ff9ade0b3065ba974cd385",
+                "sha256:37c5028cebdf731298724070838fb3a71ef1fbd201d193d311ac2cbdbca25a23",
+                "sha256:38a0e7ee65c8999394d92d9c724434cb629279d19844f2b69d9bbc46dc8b8b61",
+                "sha256:39aa09ed7ce2a648c904f79032d16dda29e6913112af8465a7bf710eef23c7ca",
+                "sha256:3cd7ee8bbfab277ab56e272221886fd33a1b5943fbf45ae9195aa6a48715a8a0",
+                "sha256:3d642e5c029e2acfacf6aa0a7a3e822086b3b777c70d364742561f9ca64c1ffc",
+                "sha256:41bbc2678a5b6a19371b2cb51f30ccea71f0c14b26477d2d884fed761cea42c7",
+                "sha256:45327fc57afbe3f2c3d7f54a335d5cecee8a9fdb3906a2fbed8af4092f4926df",
+                "sha256:4542c98b8364b976593703a2dda97377433b102f380b61bc3a2cbc2fbdae1d1f",
+                "sha256:45fa1e8ad6f4367ad73674ca560da8e827cc890eaf371f3ee063d6d7366a207b",
+                "sha256:4638ebc17de08c2f3acba557efeb6f195c88b7299d8c55c0bb4e20638bbd4d03",
+                "sha256:464bf799b422be662e5e562e62beeffc9eaa907d381a9d63a2556615bbda286d",
+                "sha256:4788135db4bd83a5edc3522b11544b013be7d25b74b155e08dd3b20cd6663bbb",
+                "sha256:47e8f034be31390a8f525431eb5e803a78ce7e2e11b32abf5361a972e14e6b61",
+                "sha256:4824eb018f0a4680b1e434697a9bf3f41c7799b80076d06530cbbd212e040ccc",
+                "sha256:4bf20c9722821fce766e685718e739deeccc60d6bc7be5029281db41f999ee0c",
+                "sha256:4d3097c39d7d4e8dba2ef86de171dcccad876c36d8379415ba18a5a4d0533510",
+                "sha256:4d889d498fce64bfcd8adf1a78579a7f626f825cbeb2956a24a29b35f9a1df32",
+                "sha256:4d965c7c4b40d1cedec9188782e98bd576f9a04868835604200c3a6e817b824f",
+                "sha256:4e26944e64ecc1d7b19db954c0f7b471f3b141ec8e1a9f57cfe27671525cd248",
+                "sha256:534f3f63c000f08050c6f7f4378bf2b52d7ba9214e9d35e3f60f7ad24a4d6425",
+                "sha256:539432f911686cb80284c30b33eaf9f4fd9a11e1111fe0dc98fdbdce69b49821",
+                "sha256:5af2d43b1978958d91351afbcc9b4d0cfe144c46c61740e82aaac8bb39ab1a4d",
+                "sha256:5cfb5ac4e82c47d5dc25b209dd4c3989e284b80109f9e08b33c895080c424b4f",
+                "sha256:616b3451b05ca63b8f433c627f68046b39543faeaa4e50d8c6699a2a1e4b85a5",
+                "sha256:6441a29f42585f085db0c04cd0557d4cbbb46fa68a0972409b1cfe9f430280c1",
+                "sha256:64bfd2c35a2c350f73ac52dc134d8775f93359c4c969280a6fe5301b5b6e7431",
+                "sha256:6ca34c29fbd6592de5fd39e80c1993634d704c4e7e14ba54c87b2c7c53da68fe",
+                "sha256:73929a2fb600a2333fce2efd92596cff5e6bf8946e20e93c067b220760064862",
+                "sha256:73f62bb7fd862d9bcd886e10612bade6fe042eda8b47e8c129892bcfb7b45e84",
+                "sha256:7584171eb3115acd4aba699bc836634783f5bd5aab131e88d8eeb8a3328a4a72",
+                "sha256:78b1ac0151271ce62bc2b33755f1043eda6a310373143a2f27e2bcd3d5fc8633",
+                "sha256:7cb496e934b71f1ade844ab91d6ccac78a3520e5df02fdb2357f85a71e541e69",
+                "sha256:7d55e38a89ec2ae17b2fa7ffeda6b70f63afab1888bd0d57aaa7b7879760acb4",
+                "sha256:7ecf0a67b212900e92f328181fed02840d74ed39553cdb38d27314e2b9c89dfa",
+                "sha256:85cd9c0af34e371390e3cb2f3a470b0b40cc07568c1e966c638c49062be6352d",
+                "sha256:8ba3073eb38a1294e8c7902989fb80a7a147a69db2396818722bd078476586a0",
+                "sha256:8d0dbcc57839831ae79fd24b1b83d42bc9448d79feaf3ed3fb5cbf94ffbf3eb7",
+                "sha256:9342de50824b40f55d2600f66c6f9a91a3a24851eca39145a749a3dc804ee599",
+                "sha256:937c0fe9538f1212b62df6a68f8d78df3572fe3682d9a0dd8851eac8a4e46063",
+                "sha256:9eff3837d447fccf2ac38c259b14ab9cbde700df355a45a1f3ff244d5e78f8b6",
+                "sha256:9ff322c7e1030543d35d83bb521b69114d3d150750528d7757544f639def9ad6",
+                "sha256:a3e9a18401a28db4358da2e191508702dbf065f2664c710708cdf9552b9fa50c",
+                "sha256:a439fd0d45d51245bbde799726adda5bd18aed3fa2b01ab2e6a64d6d13776fa3",
+                "sha256:a666134b41712e30a71afaa26deeb4da374179f769fa49784cdf0e7698880fab",
+                "sha256:ad442b8585ed4a3c2d22e4bf7b465d9b7d281e055b09719a8aeb5b576422dc9b",
+                "sha256:ad46027dbd5c1db87dc0b49becbe23093b143a20302028d387dae37ee5ef95f5",
+                "sha256:ad814864aba263be9c83ada44a95f72d10caabbf91589321f95c29c902bdcff0",
+                "sha256:adcb9c8848e15c613e483e0b99767ae325af27fe0dbd866df01fe5849d06e6e1",
+                "sha256:af693a89db6d6ac97dd84dd7769b3f2bd9007b578127d0e7dda03053f4d3b34b",
+                "sha256:afa8808159169368b66e4fbeafac6c6fd8f26246dc4d0dcc2caf94bd9cf1b828",
+                "sha256:ba2b807d2b62c446120906b8580cddae1d76d3de4efbb95ccc87f5e35c75b4b2",
+                "sha256:ba6a8cf089222a171b8f84e6ec2d10f7a9d14f26be3a347b14775a8741810676",
+                "sha256:bf3ed993bdf4754909f175ff348cf8f78d4451215b8aa338633f149ca3b1f37a",
+                "sha256:bf6a1d2c920cc9528e884850a4b2ee7629e3d362d5c44c66526d4097bbb07a1a",
+                "sha256:c089d8e7f1b4db08b2f8e4107304eec338df046275dad432635a9be9531e2fc8",
+                "sha256:c24465dd11b65c8510f251b095fc788c7c91481c81840112fe3f76c30793a455",
+                "sha256:cb08fab0fc1db15c277b72e33ac74ad9c0c789413da8984a3eacb22a94b42ef4",
+                "sha256:cd782807d35c8a41aaa7d30b5107784420eefd9fdc1c760d86007d43ae00b15d",
+                "sha256:d5146a6749b1905e04e62e0ad4622f079e5582f8b3abef5fb64516c623127908",
+                "sha256:dcbff997f47d45bf028bda4c3036bb3101e89a3df271281d392b6175f71c71d1",
+                "sha256:dd3b023f3317dbbbc775e43651ce1a31a9cea46216ad0b5be37afc18a2007699",
+                "sha256:deeb64335f489c3c11949cbd1d1668b3f1fb2d1c6a5bf40e126ef7bf95f9fa40",
+                "sha256:e09d9f6d722de9d4c1c5f122ea9bc6b25a05f975457805af4dcab7b0128aacbf",
+                "sha256:e33fcbea3b63a339dd94de0fc442fefacfe681cc7027ce63f67af9f7ceec7422",
+                "sha256:e3ed6834cc005798187a56c248a2240207cb8ffdda1c89e9afda4c3d526c2ea0",
+                "sha256:e4208f23f12d0ad206a07a489ef4cb15722c10b62774c4460ee4123250be938e",
+                "sha256:e427b66596a6441a5607dfc0085b47d36073f88da7ac48afd284263b9b99e6ce",
+                "sha256:e72ac299a6bf732a60852d052acf3999d234686755a02ba111e85e7ebf8155b1",
+                "sha256:ea955e4ed21f4bbb9b83fea09fc6af0bed82e69ecf6b35ec89237a0a49633033",
+                "sha256:ed5babdcd3d052ba5cf8832561f18df20778c7ccf12587b2d82f7bf3bf259a0e",
+                "sha256:eda1a89c4526826c0a87d33596a4cd15b8f58e9250f503e39af1699ba9c878e8",
+                "sha256:ef1fd1b24e9bcddcb168437686677104e205c8e25b066e73ffdf331d3bb8792b",
+                "sha256:ef6a222d54f742c24f6b143aab088702db3a827b224e75b9dd28b38597c595fe",
+                "sha256:f3dd5333049b5b3faa739e0f40b77cc8b7a1aded2f2da0e28794c81586d7b08a",
+                "sha256:f60e31e3e15e8c294bf70c60f8ae4d0c3caf3af8f26466e9aa8ea4c01302749b",
+                "sha256:f642313d559f9d9a00c4de6820124059cc3342a0d0127b18301de2c680d5ea40",
+                "sha256:f868e731a18b403b88aa434d960489ceeed0ddeb44ebc02389540731a67705e0",
+                "sha256:f93c867e5e85584a28c6a6feb6f2086d717266eb5d1210d096dd717b7f4dec04"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.1.2"
+            "version": "==2.3.0"
         },
         "pymysql": {
             "hashes": [
@@ -1674,10 +1678,10 @@
         },
         "python-crontab": {
             "hashes": [
-                "sha256:9c374d1c9d401afdd8dd958f20077f74c158ab3fffb9604296802715e887fe48",
-                "sha256:b21af4647c7bbb848fef2f020616c6b0289dcb9f94b4f991a55310ff9bec5749"
+                "sha256:6d5ba3c190ec76e4d252989a1644fcb233dbf53fbc8fceeb9febe1657b9fb1d4",
+                "sha256:79fb7465039ddfd4fb93d072d6ee0d45c1ac8bf1597f0686ea14fd4361dba379"
             ],
-            "version": "==2.7.1"
+            "version": "==3.0.0"
         },
         "python-dateutil": {
             "hashes": [
@@ -1898,11 +1902,11 @@
         },
         "types-awscrt": {
             "hashes": [
-                "sha256:5a846b77a6d35b63ccf9d33abf386c4dfc6275f9c062bc0e234c14ea8c6013a9",
-                "sha256:7edec4283fec9f92574444ce45fc0be65c846056634628aee22c118712da71c4"
+                "sha256:6cdfc458aca186fb73261895efeba1a2a0f3cf8f63f8b6ef32ec43739a64e4ae",
+                "sha256:cd0d4313fac0859a2b406080782076648524840e26780dbd3b5cbd9cc315834a"
             ],
-            "markers": "python_version >= '3.7' and python_version < '4.0'",
-            "version": "==0.16.23"
+            "markers": "python_version >= '3.7' and python_version < '4'",
+            "version": "==0.16.24"
         },
         "types-pymysql": {
             "hashes": [
@@ -1963,7 +1967,7 @@
                 "sha256:6d1ac1dedac750d570428362acdf60fdd4f277b0788855c3894d3226756b2bfb",
                 "sha256:75ac1d7143d58c1e6af467cfd4a96c67ee058a3adf7c249d9309999e1f5f41e4"
             ],
-            "markers": "python_version >= '3.7' and python_version < '4.0'",
+            "markers": "python_version >= '3.7' and python_version < '4'",
             "version": "==0.6.1"
         },
         "types-urllib3": {

From 422ab41a5aef3b2f61064dd5cedf51da743d9f32 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Mon, 17 Jul 2023 11:07:42 +0800
Subject: [PATCH 145/161] docs: change schema to chinese

---
 dongtai_protocol/views/agent_download.py      |  2 +-
 dongtai_protocol/views/health_oss.py          |  8 +++---
 dongtai_protocol/views/hook_profiles.py       |  2 +-
 dongtai_web/aggr_vul/aggr_vul_list.py         | 19 ++++---------
 dongtai_web/aggr_vul/aggr_vul_summary.py      |  5 ++--
 dongtai_web/aggregation/aggregation_del.py    |  6 ++---
 .../aggregation/aggregation_project_del.py    |  7 ++---
 dongtai_web/enum/hook_rules.py                | 15 ++---------
 dongtai_web/header_vul/base.py                | 14 +++-------
 dongtai_web/views/project_summary.py          |  8 +-----
 dongtai_web/views/sensitive_info_rule.py      | 27 +++++++------------
 dongtai_web/views/user_detail.py              |  2 +-
 dongtai_web/views/user_token.py               |  4 +--
 13 files changed, 34 insertions(+), 85 deletions(-)

diff --git a/dongtai_protocol/views/agent_download.py b/dongtai_protocol/views/agent_download.py
index f1c020f50..2981e83bd 100644
--- a/dongtai_protocol/views/agent_download.py
+++ b/dongtai_protocol/views/agent_download.py
@@ -306,7 +306,7 @@ def make_download_handler(self, language, user_id):
 
     @extend_schema(operation_id="agent download api",
                    tags=[_('Agent Protocol')],
-                   summary=_('Agent download'),  # type: ignore
+                   summary="Agent 下载",
                    parameters=[
                        DongTaiParameter.OPENAPI_URL,
                        DongTaiParameter.PROJECT_NAME,
diff --git a/dongtai_protocol/views/health_oss.py b/dongtai_protocol/views/health_oss.py
index ed054aae4..f1d935b7d 100644
--- a/dongtai_protocol/views/health_oss.py
+++ b/dongtai_protocol/views/health_oss.py
@@ -6,12 +6,10 @@
 # @description :
 ######################################################################
 
-import oss2
 from drf_spectacular.utils import extend_schema
-from oss2.exceptions import RequestError
 import logging
-from dongtai_protocol.utils import checkossstatus, STATUSMAP
-from dongtai_common.endpoint import OpenApiEndPoint, R, UserEndPoint
+from dongtai_protocol.utils import checkossstatus
+from dongtai_common.endpoint import R, UserEndPoint
 
 logger = logging.getLogger("dongtai.openapi")
 
@@ -21,7 +19,7 @@ class OSSHealthView(UserEndPoint):
         description='Check OSS Health',
         responses=R,
         methods=['GET'],
-        summary="Check OSS Health",
+        summary="检查 OSS 健康",
         tags=["OSS"],
     )
     def get(self, request):
diff --git a/dongtai_protocol/views/hook_profiles.py b/dongtai_protocol/views/hook_profiles.py
index 2f5e2b1f9..f7852e30e 100644
--- a/dongtai_protocol/views/hook_profiles.py
+++ b/dongtai_protocol/views/hook_profiles.py
@@ -120,7 +120,7 @@ def get_profiles(user=None, language_id=JAVA, full=False, system_only=False):
         ],
         responses=R,
         methods=['GET'],
-        summary="Pull Agent Engine Hook Rule",
+        summary="拉取 Agent Engine Hook Rule",
         tags=['Agent服务端交互协议'],
     )
     def get(self, request):
diff --git a/dongtai_web/aggr_vul/aggr_vul_list.py b/dongtai_web/aggr_vul/aggr_vul_list.py
index b6f209d6d..91480808f 100644
--- a/dongtai_web/aggr_vul/aggr_vul_list.py
+++ b/dongtai_web/aggr_vul/aggr_vul_list.py
@@ -1,33 +1,24 @@
 # 按类型获取 组件漏洞 应用漏洞列表
 from typing import Any
-from elasticsearch_dsl import Q, Search
+from elasticsearch_dsl import Q
 from dongtai_common.models.asset_vul import IastAssetVulnerabilityDocument
 from dongtai_common.common.utils import make_hash
 from dongtai_conf import settings
 from django.core.cache import cache
-from dongtai_common.models.vul_level import IastVulLevel
-from dongtai_common.models.project import IastProject
-from dongtai_common.models.program_language import IastProgramLanguage
-from dongtai_common.models.vulnerablity import IastVulnerabilityStatus
-from dongtai_common.models.strategy import IastStrategyModel
-from elasticsearch_dsl import A
 from elasticsearch import Elasticsearch
-import json
-import time
 import logging
 from dongtai_common.endpoint import R
-from django.forms import model_to_dict
 from dongtai_common.endpoint import UserEndPoint
 
 from dongtai_web.utils import extend_schema_with_envcheck
 from dongtai_web.serializers.aggregation import AggregationArgsSerializer
 from rest_framework.serializers import ValidationError
 from django.utils.translation import gettext_lazy as _
-from dongtai_web.aggregation.aggregation_common import getAuthUserInfo, turnIntListOfStr, getAuthBaseQuery, auth_user_list_str
+from dongtai_web.aggregation.aggregation_common import turnIntListOfStr, auth_user_list_str
 import pymysql
 from dongtai_web.serializers.vul import VulSerializer
-from dongtai_common.models.asset_vul import IastAssetVul, IastVulAssetRelation, IastAssetVulType, IastAssetVulTypeRelation
-from dongtai_common.models import AGGREGATION_ORDER, LANGUAGE_ID_DICT, SHARE_CONFIG_DICT, APP_LEVEL_RISK, LICENSE_RISK, \
+from dongtai_common.models.asset_vul import IastAssetVul, IastVulAssetRelation, IastAssetVulTypeRelation
+from dongtai_common.models import AGGREGATION_ORDER, LANGUAGE_ID_DICT, APP_LEVEL_RISK, LICENSE_RISK, \
     SCA_AVAILABILITY_DICT
 from dongtai_conf.settings import ELASTICSEARCH_STATE
 from typing import List
@@ -62,7 +53,7 @@ class GetAggregationVulList(UserEndPoint):
     @extend_schema_with_envcheck(
         request=AggregationArgsSerializer,
         tags=[_('VulList')],
-        summary=_('Vul List Select'),
+        summary=_('组件漏洞列表'),
         description=_("select sca vul and app vul by keywords"),
     )
     # 组件漏洞 列表
diff --git a/dongtai_web/aggr_vul/aggr_vul_summary.py b/dongtai_web/aggr_vul/aggr_vul_summary.py
index 34cacbb90..f750a99a7 100644
--- a/dongtai_web/aggr_vul/aggr_vul_summary.py
+++ b/dongtai_web/aggr_vul/aggr_vul_summary.py
@@ -6,14 +6,13 @@
 from dongtai_web.serializers.aggregation import AggregationArgsSerializer
 from django.utils.translation import gettext_lazy as _
 from dongtai_common.endpoint import R
-from dongtai_web.aggregation.aggregation_common import auth_user_list_str
 from dongtai_common.models import LANGUAGE_DICT
 from rest_framework.serializers import ValidationError
 from django.db import connection
 from dongtai_common.common.utils import cached_decorator
 from dongtai_common.models import APP_LEVEL_RISK
 from dongtai_common.models.user import User
-from typing import Dict, Union, List
+from typing import List
 from typing import TypedDict
 
 class Level(TypedDict):
@@ -322,7 +321,7 @@ class GetScaSummary(UserEndPoint):
     @extend_schema_with_envcheck(
         request=AggregationArgsSerializer,
         tags=[_('VulList')],
-        summary=_('Vul List Select'),
+        summary=_('组件漏洞列表'),
         description=_(
             "count sca vul and app vul by keywords"
         ),
diff --git a/dongtai_web/aggregation/aggregation_del.py b/dongtai_web/aggregation/aggregation_del.py
index a8e9a4384..f2b7dcb5c 100644
--- a/dongtai_web/aggregation/aggregation_del.py
+++ b/dongtai_web/aggregation/aggregation_del.py
@@ -4,10 +4,8 @@
 from dongtai_common.models.asset_vul import IastVulAssetRelation
 
 from dongtai_common.models.vulnerablity import IastVulnerabilityModel
-from dongtai_common.models.asset import Asset
 from django.utils.translation import gettext_lazy as _
-from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
-from django.db import connection
+from dongtai_web.utils import extend_schema_with_envcheck
 from dongtai_web.aggregation.aggregation_common import turnIntListOfStr
 import logging
 
@@ -21,7 +19,7 @@ class DelVulMany(UserEndPoint):
     @extend_schema_with_envcheck(
 
         tags=[_('VulList')],
-        summary=_('Vul List delete'),
+        summary=_('删除漏洞列表'),
         description=_(
             "delete many app vul and dongtai_sca vul"
         ),
diff --git a/dongtai_web/aggregation/aggregation_project_del.py b/dongtai_web/aggregation/aggregation_project_del.py
index a8fe5de06..1365f5fa1 100644
--- a/dongtai_web/aggregation/aggregation_project_del.py
+++ b/dongtai_web/aggregation/aggregation_project_del.py
@@ -4,11 +4,8 @@
 from dongtai_common.models.asset_vul import IastVulAssetRelation
 
 from dongtai_common.models.vulnerablity import IastVulnerabilityModel
-from dongtai_common.models.asset import Asset
 from django.utils.translation import gettext_lazy as _
-from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
-from django.db import connection
-from dongtai_web.aggregation.aggregation_common import turnIntListOfStr
+from dongtai_web.utils import extend_schema_with_envcheck
 import logging
 
 logger = logging.getLogger('dongtai-dongtai_conf')
@@ -20,7 +17,7 @@ class DelVulProjectLevel(UserEndPoint):
 
     @extend_schema_with_envcheck(
         tags=[_('VulList')],
-        summary=_('Vul List delete'),
+        summary=_('删除 Vul List'),
         description=_("delete many app vul and dongtai_sca vul"),
     )
     def post(self, request):
diff --git a/dongtai_web/enum/hook_rules.py b/dongtai_web/enum/hook_rules.py
index 202f27ac9..ab492600a 100644
--- a/dongtai_web/enum/hook_rules.py
+++ b/dongtai_web/enum/hook_rules.py
@@ -1,20 +1,9 @@
 import logging
-import json
 
-from django.db.models import Q, F, Count
 from django.utils.translation import gettext_lazy as _
 from dongtai_common.endpoint import UserEndPoint, R
-from rest_framework.viewsets import ViewSet
-from dongtai_common.models.agent import IastAgent
-from dongtai_common.models.strategy import IastStrategyModel
-from dongtai_common.models.vulnerablity import IastVulnerabilityModel
-from django.core.cache import cache
-from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
-from dongtai_common.models.dast_integration import IastDastIntegration
-from rest_framework import serializers
+from dongtai_web.utils import extend_schema_with_envcheck
 from rest_framework import viewsets
-from rest_framework.serializers import ValidationError
-from dongtai_common.models.profile import IastProfile
 from dongtai_conf.settings import DEFAULT_TAINT_VALUE_RANGE_COMMANDS, DEFAULT_IAST_VALUE_TAG
 
 logger = logging.getLogger('dongtai-webapi')
@@ -22,7 +11,7 @@
 
 class HookRuleEnumEndPoint(UserEndPoint, viewsets.ViewSet):
 
-    @extend_schema_with_envcheck(summary=_('Hook Rule Enums'),
+    @extend_schema_with_envcheck(summary=_('Hook Rule 枚举'),
                                  description=_("Hook Rule Enums "),
                                  tags=[_('Hook Rule')])
     def get_enums(self, request):
diff --git a/dongtai_web/header_vul/base.py b/dongtai_web/header_vul/base.py
index f91d481e7..f7ae25e6f 100644
--- a/dongtai_web/header_vul/base.py
+++ b/dongtai_web/header_vul/base.py
@@ -1,20 +1,12 @@
 import logging
 
 from dongtai_common.endpoint import UserEndPoint, R
-from dongtai_common.models.hook_type import HookType
-from dongtai_common.utils import const
 
-from dongtai_web.serializers.hook_type_strategy import HookTypeSerialize
 from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
-from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
-from django.utils.text import format_lazy
+from dongtai_web.utils import extend_schema_with_envcheck
 from rest_framework.serializers import ValidationError
-from dongtai_web.serializers.hook_strategy import HOOK_TYPE_CHOICE
 from rest_framework import viewsets
-from django.db import connection
-from django.db import models
-from dongtai_common.permissions import TalentAdminPermission
 from dongtai_common.models.header_vulnerablity import IastHeaderVulnerability, IastHeaderVulnerabilityDetail
 from django.db.models import Q
 from typing import Dict
@@ -62,7 +54,7 @@ def get_permissions(self):
     @extend_schema_with_envcheck(
         [HeaderVulArgsSerializer],
         tags=[_('Header Vul')],
-        summary=_('Header Vul List'),
+        summary=_('Header Vul 列表'),
         description=_("Get the item corresponding to the user"),
     )
     def list(self, request):
@@ -84,7 +76,7 @@ def list(self, request):
 
     @extend_schema_with_envcheck(
         tags=[_('Header Vul')],
-        summary=_('Header Vul delete'),
+        summary=_('Header Vul 删除'),
         description=_("Get the item corresponding to the user"),
     )
     def delete(self, request, pk):
diff --git a/dongtai_web/views/project_summary.py b/dongtai_web/views/project_summary.py
index 11fa05173..4f4979c79 100644
--- a/dongtai_web/views/project_summary.py
+++ b/dongtai_web/views/project_summary.py
@@ -9,17 +9,11 @@
 from dongtai_common.models.agent import IastAgent
 from dongtai_common.models.project import IastProject
 from dongtai_common.models.project_version import IastProjectVersion
-from dongtai_common.models.vul_level import IastVulLevel
-from dongtai_common.models.vulnerablity import IastVulnerabilityModel
 from dongtai_web.base.project_version import get_project_version, get_project_version_by_id, ProjectsVersionDataSerializer
 from django.utils.translation import gettext_lazy as _
-from dongtai_common.models.vulnerablity import IastVulnerabilityStatus
 from dongtai_web.serializers.project import ProjectSerializer
-from dongtai_common.models.hook_type import HookType
-from django.db.models import Q
 from rest_framework import serializers
 from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
-from dongtai_common.models.strategy import IastStrategyModel
 from dongtai_web.views.utils.commonstats import get_summary_by_project
 from dongtai_common.utils import const
 
@@ -97,7 +91,7 @@ def weeks_ago(week=1):
 
     @extend_schema_with_envcheck(
         tags=[_('Project')],
-        summary=_('Projects Summary'),
+        summary=_('项目总结'),
         description=_("Get project deatils and its statistics data about vulnerablity."
                       ),
         response_schema=_ProjectSummaryResponseSerializer,
diff --git a/dongtai_web/views/sensitive_info_rule.py b/dongtai_web/views/sensitive_info_rule.py
index a269bf35f..351a1e442 100644
--- a/dongtai_web/views/sensitive_info_rule.py
+++ b/dongtai_web/views/sensitive_info_rule.py
@@ -13,7 +13,6 @@
     BatchStatusUpdateSerializerView,
     AllStatusUpdateSerializerView,
 )
-from dongtai_common.permissions import TalentAdminPermission
 try:
     import re2 as re
 except ImportError as e:
@@ -25,24 +24,16 @@
 from dongtai_common.models.sensitive_info import IastPatternType, IastSensitiveInfoRule
 from django.db.models import Q
 import time
-from dongtai_common.models.user import User
 from dongtai_common.models.strategy import IastStrategyModel
-from django.db import models
 import logging
 
 from dongtai_common.endpoint import UserEndPoint, R
-from dongtai_common.models.hook_type import HookType
-from dongtai_common.utils import const
 
-from dongtai_web.serializers.hook_type_strategy import HookTypeSerialize
 from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
-from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
-from django.utils.text import format_lazy
+from dongtai_web.utils import extend_schema_with_envcheck
 from rest_framework.serializers import ValidationError
-from dongtai_web.serializers.hook_strategy import HOOK_TYPE_CHOICE
 from rest_framework import viewsets
-from django.db import connection
 
 logger = logging.getLogger('dongtai-webapi')
 
@@ -168,7 +159,7 @@ def list(self, request):
     @extend_schema_with_envcheck(
         request=SensitiveInfoRuleCreateSerializer,
         tags=[_('SensitiveInfoRule')],
-        summary=_('SensitiveInfoRule Create'),
+        summary=_('敏感信息规则创建'),
         description=_("Get the item corresponding to the user, support fuzzy search based on name."
                       ),
     )
@@ -202,7 +193,7 @@ def create(self, request):
     @extend_schema_with_envcheck(
         request=SensitiveInfoRuleCreateSerializer,
         tags=[_('SensitiveInfoRule')],
-        summary=_('SensitiveInfoRule Update'),
+        summary=_('敏感信息规则更新'),
         description=_("Get the item corresponding to the user, support fuzzy search based on name."
                       ),
     )
@@ -224,7 +215,7 @@ def update(self, request, pk):
 
     @extend_schema_with_envcheck(
         tags=[_('SensitiveInfoRule')],
-        summary=_('SensitiveInfoRule delete'),
+        summary=_('敏感信息规则删除'),
         description=_("Get the item corresponding to the user, support fuzzy search based on name."
                       ),
     )
@@ -236,7 +227,7 @@ def destory(self, request, pk):
 
     @extend_schema_with_envcheck(
         tags=[_('SensitiveInfoRule')],
-        summary=_('SensitiveInfoRule get'),
+        summary=_('敏感信息规则获取'),
         description=_("Get the item corresponding to the user, support fuzzy search based on name."
                       ),
     )
@@ -253,7 +244,7 @@ class SensitiveInfoPatternTypeView(UserEndPoint):
 
     @extend_schema_with_envcheck(
         tags=[_('SensitiveInfoRule')],
-        summary=_('SensitiveInfoRule Pattern Type List'),
+        summary=_('敏感信息规则模式类型列表'),
         description=_("Get the item corresponding to the user."
                       ),
     )
@@ -266,7 +257,7 @@ class SensitiveInfoPatternValidationView(UserEndPoint):
     @extend_schema_with_envcheck(
         request=_RegexPatternValidationSerializer,
         tags=[_('SensitiveInfoRule')],
-        summary=_('SensitiveInfoRule validated_data'),
+        summary=_('敏感信息规则数据验证'),
         description=_("Get the item corresponding to the user, support fuzzy search based on name."
                       ),
     )
@@ -339,7 +330,7 @@ class SensitiveInfoRuleBatchView(BatchStatusUpdateSerializerView):
     @extend_schema_with_envcheck(
         request=BatchStatusUpdateSerializerView.serializer,
         tags=[_('SensitiveInfoRule')],
-        summary=_('SensitiveInfoRule batch status'),
+        summary=_('敏感信息规则状态批量更新'),
         description=_("batch update status."),
     )
     def post(self, request):
@@ -356,7 +347,7 @@ class SensitiveInfoRuleAllView(AllStatusUpdateSerializerView):
     @extend_schema_with_envcheck(
         request=AllStatusUpdateSerializerView.serializer,
         tags=[_('SensitiveInfoRule')],
-        summary=_('SensitiveInfoRule all status'),
+        summary=_('敏感信息规则全部状态更新'),
         description=_("all update status."),
     )
     def post(self, request):
diff --git a/dongtai_web/views/user_detail.py b/dongtai_web/views/user_detail.py
index ca1779a40..ed5c614bb 100644
--- a/dongtai_web/views/user_detail.py
+++ b/dongtai_web/views/user_detail.py
@@ -14,7 +14,7 @@
 
 class UserDetailEndPoint(TalentAdminEndPoint):
     @extend_schema(
-        summary=_("User Detail"),
+        summary=_("用户详情"),
         tags=[_("User")],
     )
     def get(self, request, user_id):
diff --git a/dongtai_web/views/user_token.py b/dongtai_web/views/user_token.py
index 6d84041fa..d9f961676 100644
--- a/dongtai_web/views/user_token.py
+++ b/dongtai_web/views/user_token.py
@@ -32,10 +32,10 @@ def get(self, request):
 
 class UserDepartmentToken(UserEndPoint):
     name = "iast-v1-user-department-token"
-    description = _("Get Department Deploy token")
+    description = _("获取部门部署 token")
 
     @extend_schema(
-        summary=_("Get Department Deploy token"),
+        summary=_("获取部门部署 token"),
         tags=[_("User")],
     )
     def get(self, request):

From 1f9d1004b45bfeec85865774774bec36faa7a3ea Mon Sep 17 00:00:00 2001
From: tscuite 
Date: Mon, 17 Jul 2023 11:52:04 +0800
Subject: [PATCH 146/161] feat: update ci

---
 .github/workflows/deploy-dev.yaml             |  2 +-
 deploy/docker/entrypoint.sh                   |  2 ++
 deploy/kubernetes/helm/templates/_helpers.tpl | 22 +++++++++++++++++++
 .../templates/deployments/dongtai-server.yaml |  7 ++++++
 4 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/deploy-dev.yaml b/.github/workflows/deploy-dev.yaml
index 6d49b6853..540b8751e 100644
--- a/.github/workflows/deploy-dev.yaml
+++ b/.github/workflows/deploy-dev.yaml
@@ -96,4 +96,4 @@ jobs:
             git clone https://github.com/HXSecurity/DongTai.git
             helm upgrade --install huoxian --create-namespace -n iast-${{ env.helm_ns }} ./DongTai/deploy/kubernetes/helm/ \
             --set sca.sca_token=${{ secrets.TOKEN_SCA }} --set usb.usb_token=${{ secrets.TOKEN_SCA }} --set mysql.host=iast-mysql-${{ env.helm_mysql }}.huoxian.cn \
-            --set tag=${{ steps.version.outputs.GITHUB_REF }}-latest --set build.server_number=iast${{github.run_number}} --values https://charts.dongtai.io/devops.yaml
\ No newline at end of file
+            --set tag=${{ steps.version.outputs.GITHUB_REF }}-latest --set build.server_number=iast${{github.run_number}} --set develop.agentZip=${{ env.helm_ns }} --values https://charts.dongtai.io/devops.yaml
\ No newline at end of file
diff --git a/deploy/docker/entrypoint.sh b/deploy/docker/entrypoint.sh
index b566347aa..d1bc20bb4 100755
--- a/deploy/docker/entrypoint.sh
+++ b/deploy/docker/entrypoint.sh
@@ -23,7 +23,9 @@ elif [ "$1" = "beat" ]; then
 elif [ "$1" = "healthcheck" ]; then
   celery -A dongtai_conf inspect ping -d celery@$(hostname)
 else
+  if [ ! -n "${2}" ]; then
 	echo "Get the latest vulnerability rules." && python manage.py load_hook_strategy
 	if [ $? -ne 0 ]; then echo "ERROR: Lost connection to MySQL server !!!" && exit 1 ; else echo "succeed" ;fi
+	fi
 	uwsgi --ini /opt/dongtai/dongtai_conf/conf/uwsgi.ini $DONGTAI_CONCURRENCY
 fi
diff --git a/deploy/kubernetes/helm/templates/_helpers.tpl b/deploy/kubernetes/helm/templates/_helpers.tpl
index 8cf0b1c4a..477a9ae98 100644
--- a/deploy/kubernetes/helm/templates/_helpers.tpl
+++ b/deploy/kubernetes/helm/templates/_helpers.tpl
@@ -94,6 +94,10 @@ volumeMounts:
   - name: {{ template "dongtai.fullname" . }}-configfile
     mountPath: /opt/dongtai/dongtai_conf/conf/config.ini
     subPath: config.ini
+{{- if .Values.develop.dev }}
+  - name: dongtai-tmp-volumezip
+    mountPath: /tmp/iast_cache/package
+{{- end -}}
 {{- if .Values.storage.persistentVolumeClaim }}
   - name: {{ template "dongtai.fullname" . }}-log-path
     mountPath: /tmp/logstash
@@ -129,6 +133,20 @@ volumeMounts:
 {{- define "deploy.imagePullPolicy" -}}
 imagePullPolicy: {{.Values.imagePullPolicy}}
 {{- end -}}
+{{- define "deploy.devinitContainers" -}}
+initContainers:
+  - name: init-dongtai-agentcontainer
+    command: ["/bin/sh", "-c"]
+    args: ["cd /tmp/iast_cache/package && curl -s https://charts.dongtai.io/agent_{{.Values.develop.agentZip}}/java/latest/agent_latest.tar.gz | tar -xvzf -"]
+    image: curlimages/curl
+    imagePullPolicy: Always
+    resources: {}
+    terminationMessagePath: /dev/termination-log
+    terminationMessagePolicy: File
+    volumeMounts:
+    - name: dongtai-tmp-volumezip
+      mountPath: /tmp/iast_cache/package
+{{- end -}}
 
 {{- define "deploy.initContainers" -}}
 initContainers:
@@ -193,6 +211,10 @@ volumes:
     persistentVolumeClaim:
       {{ include "deploy.config.persistentVolumeClaim" . }}
 {{- end -}}
+{{- if .Values.develop.dev }}
+  - name: dongtai-tmp-volumezip
+    emptyDir: {}
+{{- end -}}
 {{- end -}}
 
 {{- define "deploy.config.persistentVolumeClaim" -}}
diff --git a/deploy/kubernetes/helm/templates/deployments/dongtai-server.yaml b/deploy/kubernetes/helm/templates/deployments/dongtai-server.yaml
index faf9e0218..66b21993d 100644
--- a/deploy/kubernetes/helm/templates/deployments/dongtai-server.yaml
+++ b/deploy/kubernetes/helm/templates/deployments/dongtai-server.yaml
@@ -34,6 +34,10 @@ spec:
       containers:
         - name: {{ template "dongtai.fullname" . }}-server-container
           image: {{ .Values.images }}/dongtai-server:{{ .Values.tag }}
+          command: [ "/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh" ]
+          {{- if .Values.develop.dev }}
+          args: ["server","server"]
+          {{- end }}
           env:
             - name: DONGTAI_CONCURRENCY
               value: {{.Values.build.env_server}}
@@ -64,6 +68,9 @@ spec:
               scheme: HTTP
           {{- end }}
       {{- include "deploy.config.vo" . | nindent 6 }}
+      {{- if .Values.develop.dev }}
+      {{- include "deploy.devinitContainers" . | nindent 6 }}
+      {{- end }}
       {{- if .Values.somaxconn }}
       {{- include "deploy.initContainers" . | nindent 6 }}
       {{- end }}
\ No newline at end of file

From 5edb19ccd7ee39ff03fb86a9f93db3c01bbd72a4 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Mon, 17 Jul 2023 12:29:43 +0800
Subject: [PATCH 147/161] docs: change schema to chinese

---
 dongtai_common/endpoint/__init__.py           |  1 +
 dongtai_protocol/views/agent_download.py      |  3 +-
 dongtai_web/aggr_vul/aggr_vul_list.py         |  2 +-
 dongtai_web/aggr_vul/aggr_vul_summary.py      |  2 +-
 dongtai_web/aggregation/aggregation_del.py    |  2 +-
 .../aggregation/aggregation_project_del.py    |  2 +-
 dongtai_web/views/sensitive_info_rule.py      | 18 +++++-----
 .../vul_recheck_payload.py                    | 35 +++++++++++++++++++
 8 files changed, 50 insertions(+), 15 deletions(-)

diff --git a/dongtai_common/endpoint/__init__.py b/dongtai_common/endpoint/__init__.py
index 58ee0a610..f475cb83f 100644
--- a/dongtai_common/endpoint/__init__.py
+++ b/dongtai_common/endpoint/__init__.py
@@ -137,6 +137,7 @@ def dispatch(self, request, *args, **kwargs):
 
                 if operate_method == "GET":
                     operate_type = OperateType.GET
+                    return self.response
                 elif operate_method == "POST":
                     operate_type = OperateType.ADD
                 elif operate_method == "PUT":
diff --git a/dongtai_protocol/views/agent_download.py b/dongtai_protocol/views/agent_download.py
index 2981e83bd..986bd8a9a 100644
--- a/dongtai_protocol/views/agent_download.py
+++ b/dongtai_protocol/views/agent_download.py
@@ -24,7 +24,6 @@
 
 import shutil
 import tarfile
-import os
 import threading
 import time
 
@@ -305,7 +304,7 @@ def make_download_handler(self, language, user_id):
         return
 
     @extend_schema(operation_id="agent download api",
-                   tags=[_('Agent Protocol')],
+                   tags=[_('Agent服务端交互协议')],
                    summary="Agent 下载",
                    parameters=[
                        DongTaiParameter.OPENAPI_URL,
diff --git a/dongtai_web/aggr_vul/aggr_vul_list.py b/dongtai_web/aggr_vul/aggr_vul_list.py
index 91480808f..7c8341ba1 100644
--- a/dongtai_web/aggr_vul/aggr_vul_list.py
+++ b/dongtai_web/aggr_vul/aggr_vul_list.py
@@ -52,7 +52,7 @@ class GetAggregationVulList(UserEndPoint):
 
     @extend_schema_with_envcheck(
         request=AggregationArgsSerializer,
-        tags=[_('VulList')],
+        tags=[_('漏洞')],
         summary=_('组件漏洞列表'),
         description=_("select sca vul and app vul by keywords"),
     )
diff --git a/dongtai_web/aggr_vul/aggr_vul_summary.py b/dongtai_web/aggr_vul/aggr_vul_summary.py
index f750a99a7..2047710d4 100644
--- a/dongtai_web/aggr_vul/aggr_vul_summary.py
+++ b/dongtai_web/aggr_vul/aggr_vul_summary.py
@@ -320,7 +320,7 @@ class GetScaSummary(UserEndPoint):
 
     @extend_schema_with_envcheck(
         request=AggregationArgsSerializer,
-        tags=[_('VulList')],
+        tags=[_('漏洞')],
         summary=_('组件漏洞列表'),
         description=_(
             "count sca vul and app vul by keywords"
diff --git a/dongtai_web/aggregation/aggregation_del.py b/dongtai_web/aggregation/aggregation_del.py
index f2b7dcb5c..915598162 100644
--- a/dongtai_web/aggregation/aggregation_del.py
+++ b/dongtai_web/aggregation/aggregation_del.py
@@ -18,7 +18,7 @@ class DelVulMany(UserEndPoint):
 
     @extend_schema_with_envcheck(
 
-        tags=[_('VulList')],
+        tags=[_('漏洞')],
         summary=_('删除漏洞列表'),
         description=_(
             "delete many app vul and dongtai_sca vul"
diff --git a/dongtai_web/aggregation/aggregation_project_del.py b/dongtai_web/aggregation/aggregation_project_del.py
index 1365f5fa1..9ccac3c89 100644
--- a/dongtai_web/aggregation/aggregation_project_del.py
+++ b/dongtai_web/aggregation/aggregation_project_del.py
@@ -16,7 +16,7 @@ class DelVulProjectLevel(UserEndPoint):
     description = _("del vul list of many")
 
     @extend_schema_with_envcheck(
-        tags=[_('VulList')],
+        tags=[_('漏洞')],
         summary=_('删除 Vul List'),
         description=_("delete many app vul and dongtai_sca vul"),
     )
diff --git a/dongtai_web/views/sensitive_info_rule.py b/dongtai_web/views/sensitive_info_rule.py
index 351a1e442..30562c99e 100644
--- a/dongtai_web/views/sensitive_info_rule.py
+++ b/dongtai_web/views/sensitive_info_rule.py
@@ -129,7 +129,7 @@ def get_permissions(self):
 
     @extend_schema_with_envcheck(
         [_SensitiveInfoArgsSerializer],
-        tags=[_('SensitiveInfoRule')],
+        tags=[_('敏感信息规则')],
         summary=_('SensitiveInfoRule List'),
         description=_("Get the item corresponding to the user, support fuzzy search based on name."
                       ),
@@ -158,7 +158,7 @@ def list(self, request):
 
     @extend_schema_with_envcheck(
         request=SensitiveInfoRuleCreateSerializer,
-        tags=[_('SensitiveInfoRule')],
+        tags=[_('敏感信息规则')],
         summary=_('敏感信息规则创建'),
         description=_("Get the item corresponding to the user, support fuzzy search based on name."
                       ),
@@ -192,7 +192,7 @@ def create(self, request):
 
     @extend_schema_with_envcheck(
         request=SensitiveInfoRuleCreateSerializer,
-        tags=[_('SensitiveInfoRule')],
+        tags=[_('敏感信息规则')],
         summary=_('敏感信息规则更新'),
         description=_("Get the item corresponding to the user, support fuzzy search based on name."
                       ),
@@ -214,7 +214,7 @@ def update(self, request, pk):
         return R.success(msg=_('update success'))
 
     @extend_schema_with_envcheck(
-        tags=[_('SensitiveInfoRule')],
+        tags=[_('敏感信息规则')],
         summary=_('敏感信息规则删除'),
         description=_("Get the item corresponding to the user, support fuzzy search based on name."
                       ),
@@ -226,7 +226,7 @@ def destory(self, request, pk):
         return R.success(msg=_('delete success'))
 
     @extend_schema_with_envcheck(
-        tags=[_('SensitiveInfoRule')],
+        tags=[_('敏感信息规则')],
         summary=_('敏感信息规则获取'),
         description=_("Get the item corresponding to the user, support fuzzy search based on name."
                       ),
@@ -243,7 +243,7 @@ def retrieve(self, request, pk):
 class SensitiveInfoPatternTypeView(UserEndPoint):
 
     @extend_schema_with_envcheck(
-        tags=[_('SensitiveInfoRule')],
+        tags=[_('敏感信息规则')],
         summary=_('敏感信息规则模式类型列表'),
         description=_("Get the item corresponding to the user."
                       ),
@@ -256,7 +256,7 @@ def get(self, request):
 class SensitiveInfoPatternValidationView(UserEndPoint):
     @extend_schema_with_envcheck(
         request=_RegexPatternValidationSerializer,
-        tags=[_('SensitiveInfoRule')],
+        tags=[_('敏感信息规则')],
         summary=_('敏感信息规则数据验证'),
         description=_("Get the item corresponding to the user, support fuzzy search based on name."
                       ),
@@ -329,7 +329,7 @@ class SensitiveInfoRuleBatchView(BatchStatusUpdateSerializerView):
 
     @extend_schema_with_envcheck(
         request=BatchStatusUpdateSerializerView.serializer,
-        tags=[_('SensitiveInfoRule')],
+        tags=[_('敏感信息规则')],
         summary=_('敏感信息规则状态批量更新'),
         description=_("batch update status."),
     )
@@ -346,7 +346,7 @@ class SensitiveInfoRuleAllView(AllStatusUpdateSerializerView):
 
     @extend_schema_with_envcheck(
         request=AllStatusUpdateSerializerView.serializer,
-        tags=[_('SensitiveInfoRule')],
+        tags=[_('敏感信息规则')],
         summary=_('敏感信息规则全部状态更新'),
         description=_("all update status."),
     )
diff --git a/dongtai_web/vul_recheck_payload/vul_recheck_payload.py b/dongtai_web/vul_recheck_payload/vul_recheck_payload.py
index a441b86cd..675ac0dc0 100644
--- a/dongtai_web/vul_recheck_payload/vul_recheck_payload.py
+++ b/dongtai_web/vul_recheck_payload/vul_recheck_payload.py
@@ -11,6 +11,7 @@
 from rest_framework import serializers
 from django.db.models import Q
 from django.forms.models import model_to_dict
+from drf_spectacular.utils import extend_schema
 
 
 def get_or_none(classmodel, function, **kwargs):
@@ -57,7 +58,15 @@ class VulReCheckPayloadViewSet(UserEndPoint, viewsets.ViewSet):
     name = "api-v1-vul-recheck-payload"
     description = _("config recheck payload V2")
 
+    @extend_schema(
+        tags=[_('主动验证')],
+        summary="增加主动验证Payload",
+        request=IastVulRecheckPayloadSerializer,
+    )
     def create(self, request):
+        """
+        增加主动验证Payload
+        """
         ser = IastVulRecheckPayloadSerializer(data=request.data)
         try:
             if ser.is_valid(True):
@@ -67,6 +76,10 @@ def create(self, request):
         vul_recheck_payload_create(ser.data, request.user.id)
         return R.success()
 
+    @extend_schema(
+        tags=[_('主动验证')],
+        summary="获取主动验证Payload",
+    )
     def retrieve(self, request, pk):
         obj = get_or_none(
             IastVulRecheckPayload,
@@ -78,6 +91,11 @@ def retrieve(self, request, pk):
             return R.failure()
         return R.success(data=obj)
 
+    @extend_schema(
+        tags=[_('主动验证')],
+        summary="获取主动验证Payload列表",
+        request=IastVulRecheckPayloadListSerializer,
+    )
     def list(self, request):
         ser = IastVulRecheckPayloadListSerializer(data=request.query_params)
         try:
@@ -106,6 +124,11 @@ def list(self, request):
         return R.success(page=page_summary,
                          data=list(page_data))
 
+    @extend_schema(
+        tags=[_('主动验证')],
+        summary="更新主动验证Payload",
+        request=IastVulRecheckPayloadSerializer,
+    )
     def update(self, request, pk):
         ser = IastVulRecheckPayloadSerializer(data=request.data)
         try:
@@ -119,6 +142,10 @@ def update(self, request, pk):
             return R.success()
         return R.success()
 
+    @extend_schema(
+        tags=[_('主动验证')],
+        summary="删除主动验证Payload",
+    )
     def delete(self, request, pk):
         if IastVulRecheckPayload.objects.filter(
                 pk=pk, user_id=request.user.id).exists():
@@ -126,6 +153,10 @@ def delete(self, request, pk):
             return R.success()
         return R.failure()
 
+    @extend_schema(
+        tags=[_('主动验证')],
+        summary="修改主动验证Payload状态",
+    )
     def status_change(self, request):
         mode = request.data.get('mode', 1)
         q = ~Q(status=-1) & Q(user_id=request.user.id)
@@ -140,6 +171,10 @@ def status_change(self, request):
             IastVulRecheckPayload.objects.filter(q).update(status=status)
         return R.success()
 
+    @extend_schema(
+        tags=[_('主动验证')],
+        summary="批量主动验证Payload状态",
+    )
     def status_all(self, request):
         status = request.data.get('status', 0)
         q = ~Q(status=-1)

From 3f9d3b4553cf86f528ba1a7e397e98c75d8126f6 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Mon, 17 Jul 2023 14:12:12 +0800
Subject: [PATCH 148/161] fix: fix IastVulRecheckPayloadSerializer error

---
 dongtai_web/vul_recheck_payload/vul_recheck_payload.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/dongtai_web/vul_recheck_payload/vul_recheck_payload.py b/dongtai_web/vul_recheck_payload/vul_recheck_payload.py
index 675ac0dc0..8df0c3176 100644
--- a/dongtai_web/vul_recheck_payload/vul_recheck_payload.py
+++ b/dongtai_web/vul_recheck_payload/vul_recheck_payload.py
@@ -1,16 +1,10 @@
-import time
-
-from dongtai_common.endpoint import UserEndPoint, R, TalentAdminEndPoint
-from dongtai_common.models.agent_config import IastAgentConfig
+from dongtai_common.endpoint import UserEndPoint, R
 from django.utils.translation import gettext_lazy as _
-from dongtai_web.utils import extend_schema_with_envcheck, get_response_serializer
-from dongtai_web.serializers.agent_config import AgentConfigSettingSerializer
 from rest_framework.serializers import ValidationError
 from rest_framework import viewsets
 from dongtai_common.models.vul_recheck_payload import IastVulRecheckPayload
 from rest_framework import serializers
 from django.db.models import Q
-from django.forms.models import model_to_dict
 from drf_spectacular.utils import extend_schema
 
 
@@ -26,7 +20,7 @@ def get_or_none(classmodel, function, **kwargs):
 class IastVulRecheckPayloadSerializer(serializers.Serializer):
     value = serializers.CharField()
     status = serializers.IntegerField()
-    strategy_id = serializers.IntegerField()
+    strategy_id = serializers.IntegerField(min_value=1)
     language_id = serializers.IntegerField()
 
 

From ebffd15281f84627785bf83613073e2d942dd9ce Mon Sep 17 00:00:00 2001
From: st1020 
Date: Mon, 17 Jul 2023 15:04:42 +0800
Subject: [PATCH 149/161] fix: fix IastVulRecheckPayloadSerializer error

---
 dongtai_web/vul_recheck_payload/vul_recheck_payload.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/dongtai_web/vul_recheck_payload/vul_recheck_payload.py b/dongtai_web/vul_recheck_payload/vul_recheck_payload.py
index 8df0c3176..c58e2c853 100644
--- a/dongtai_web/vul_recheck_payload/vul_recheck_payload.py
+++ b/dongtai_web/vul_recheck_payload/vul_recheck_payload.py
@@ -20,16 +20,16 @@ def get_or_none(classmodel, function, **kwargs):
 class IastVulRecheckPayloadSerializer(serializers.Serializer):
     value = serializers.CharField()
     status = serializers.IntegerField()
-    strategy_id = serializers.IntegerField(min_value=1)
-    language_id = serializers.IntegerField()
+    strategy_id = serializers.IntegerField(min_value=1, max_value=2**32-1)
+    language_id = serializers.IntegerField(min_value=1, max_value=2**32-1)
 
 
 class IastVulRecheckPayloadListSerializer(serializers.Serializer):
     keyword = serializers.CharField(required=False, default=None)
     page = serializers.IntegerField()
     page_size = serializers.IntegerField()
-    strategy_id = serializers.IntegerField(required=False, default=None)
-    language_id = serializers.IntegerField(required=False, default=None)
+    strategy_id = serializers.IntegerField(required=False, default=None, min_value=1, max_value=2**32-1)
+    language_id = serializers.IntegerField(required=False, default=None, min_value=1, max_value=2**32-1)
 
 
 def vul_recheck_payload_create(data, user_id):

From de0292c488cff674df078d3d5f69e13ff56b95c1 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Mon, 17 Jul 2023 15:16:33 +0800
Subject: [PATCH 150/161] style: fix style error

---
 dongtai_web/vul_recheck_payload/vul_recheck_payload.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/dongtai_web/vul_recheck_payload/vul_recheck_payload.py b/dongtai_web/vul_recheck_payload/vul_recheck_payload.py
index c58e2c853..b06c92eb4 100644
--- a/dongtai_web/vul_recheck_payload/vul_recheck_payload.py
+++ b/dongtai_web/vul_recheck_payload/vul_recheck_payload.py
@@ -20,16 +20,16 @@ def get_or_none(classmodel, function, **kwargs):
 class IastVulRecheckPayloadSerializer(serializers.Serializer):
     value = serializers.CharField()
     status = serializers.IntegerField()
-    strategy_id = serializers.IntegerField(min_value=1, max_value=2**32-1)
-    language_id = serializers.IntegerField(min_value=1, max_value=2**32-1)
+    strategy_id = serializers.IntegerField(min_value=1, max_value=2**32 - 1)
+    language_id = serializers.IntegerField(min_value=1, max_value=2**32 - 1)
 
 
 class IastVulRecheckPayloadListSerializer(serializers.Serializer):
     keyword = serializers.CharField(required=False, default=None)
     page = serializers.IntegerField()
     page_size = serializers.IntegerField()
-    strategy_id = serializers.IntegerField(required=False, default=None, min_value=1, max_value=2**32-1)
-    language_id = serializers.IntegerField(required=False, default=None, min_value=1, max_value=2**32-1)
+    strategy_id = serializers.IntegerField(required=False, default=None, min_value=1, max_value=2**32 - 1)
+    language_id = serializers.IntegerField(required=False, default=None, min_value=1, max_value=2**32 - 1)
 
 
 def vul_recheck_payload_create(data, user_id):

From b5ad3c234c5892b076e825c8bda804f9c5717149 Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Mon, 17 Jul 2023 15:39:16 +0800
Subject: [PATCH 151/161] fix: logstash prase json error.

---
 dongtai_common/engine/vul_engine.py    | 5 +++--
 dongtai_protocol/report/log_service.py | 5 ++++-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/dongtai_common/engine/vul_engine.py b/dongtai_common/engine/vul_engine.py
index ffd039372..738cad93c 100644
--- a/dongtai_common/engine/vul_engine.py
+++ b/dongtai_common/engine/vul_engine.py
@@ -55,8 +55,9 @@ def method_pool(self, method_pool):
         self._method_pool = sorted(method_pool,
                                    key=lambda e: e.__getitem__('invokeId'),
                                    reverse=True)
-        if method_pool_is_3(method_pool[0]):
-            self._method_pool = list(map(method_pool_3_to_2, self._method_pool))
+        if method_pool and method_pool_is_3(method_pool[0]):
+            self._method_pool = list(map(method_pool_3_to_2,
+                                         self._method_pool))
             self.version = 3
         self._method_pool = sorted(self._method_pool,
                                    key=lambda e: e.__getitem__('invokeId'),
diff --git a/dongtai_protocol/report/log_service.py b/dongtai_protocol/report/log_service.py
index e0d52709f..7542ac1e4 100644
--- a/dongtai_protocol/report/log_service.py
+++ b/dongtai_protocol/report/log_service.py
@@ -1,5 +1,6 @@
 import logging
 import socket
+from base64 import b64encode
 
 logger = logging.getLogger('dongtai.openapi')
 
@@ -38,7 +39,9 @@ def send(self, message):
             if not self.socket:
                 self.create_socket()
             if self.socket:
-                self.socket.sendall(bytes(message + "\n", encoding='utf-8'), socket.MSG_DONTWAIT)
+                self.socket.sendall(
+                    b64encode(bytes(message, encoding='utf-8')) +
+                    bytes("\n", encoding='uft-8'), socket.MSG_DONTWAIT)
                 return True
         except Exception as e:
             logger.error('failed to send message to log service', exc_info=e)

From 8d44cdcd1704ccffc3d1795e05ea4549edaedef9 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Mon, 17 Jul 2023 15:56:00 +0800
Subject: [PATCH 152/161] fix: fix IastVulRecheckPayloadSerializer error

---
 dongtai_web/vul_recheck_payload/vul_recheck_payload.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/dongtai_web/vul_recheck_payload/vul_recheck_payload.py b/dongtai_web/vul_recheck_payload/vul_recheck_payload.py
index b06c92eb4..0ccc9597b 100644
--- a/dongtai_web/vul_recheck_payload/vul_recheck_payload.py
+++ b/dongtai_web/vul_recheck_payload/vul_recheck_payload.py
@@ -20,16 +20,16 @@ def get_or_none(classmodel, function, **kwargs):
 class IastVulRecheckPayloadSerializer(serializers.Serializer):
     value = serializers.CharField()
     status = serializers.IntegerField()
-    strategy_id = serializers.IntegerField(min_value=1, max_value=2**32 - 1)
-    language_id = serializers.IntegerField(min_value=1, max_value=2**32 - 1)
+    strategy_id = serializers.IntegerField(min_value=1, max_value=2**31 - 1)
+    language_id = serializers.IntegerField(min_value=1, max_value=2**31 - 1)
 
 
 class IastVulRecheckPayloadListSerializer(serializers.Serializer):
     keyword = serializers.CharField(required=False, default=None)
     page = serializers.IntegerField()
     page_size = serializers.IntegerField()
-    strategy_id = serializers.IntegerField(required=False, default=None, min_value=1, max_value=2**32 - 1)
-    language_id = serializers.IntegerField(required=False, default=None, min_value=1, max_value=2**32 - 1)
+    strategy_id = serializers.IntegerField(required=False, default=None, min_value=1, max_value=2**31 - 1)
+    language_id = serializers.IntegerField(required=False, default=None, min_value=1, max_value=2**31 - 1)
 
 
 def vul_recheck_payload_create(data, user_id):

From 61892b80e401ceb84da8fc4b9f8fc33b3a0f8529 Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Mon, 17 Jul 2023 15:57:23 +0800
Subject: [PATCH 153/161] fix: logstash prase json error.

---
 dongtai_protocol/report/log_service.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dongtai_protocol/report/log_service.py b/dongtai_protocol/report/log_service.py
index 7542ac1e4..58d9ebaa8 100644
--- a/dongtai_protocol/report/log_service.py
+++ b/dongtai_protocol/report/log_service.py
@@ -41,7 +41,7 @@ def send(self, message):
             if self.socket:
                 self.socket.sendall(
                     b64encode(bytes(message, encoding='utf-8')) +
-                    bytes("\n", encoding='uft-8'), socket.MSG_DONTWAIT)
+                    bytes("\n", encoding='utf-8'), socket.MSG_DONTWAIT)
                 return True
         except Exception as e:
             logger.error('failed to send message to log service', exc_info=e)

From 538b21f37bea35bdc77a30a90e941961ab02129f Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Mon, 17 Jul 2023 16:06:02 +0800
Subject: [PATCH 154/161] fix: logstash prase json error.

---
 dongtai_common/models/vulnerablity.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/dongtai_common/models/vulnerablity.py b/dongtai_common/models/vulnerablity.py
index 4133ace9f..1b97df31b 100644
--- a/dongtai_common/models/vulnerablity.py
+++ b/dongtai_common/models/vulnerablity.py
@@ -14,6 +14,10 @@
 from dongtai_common.models.hook_type import HookType
 from dongtai_common.models.project import IastProject
 from dongtai_common.models.project_version import IastProjectVersion
+import logging
+
+
+logger = logging.getLogger('dongtai-core')
 
 class IastVulnerabilityStatus(models.Model):
     name = models.CharField(max_length=100, blank=True)
@@ -111,8 +115,13 @@ def save(self, *args, **kwargs):
         if not self.pattern_uri:
             self.pattern_uri = self.pattern_uri
         self.search_keywords = " ".join(key_works)
-        self.latest_time_desc = -int(self.latest_time)
-        self.level_id_desc = -int(self.level_id)
+        try:
+            self.latest_time_desc = -int(self.latest_time)
+            self.level_id_desc = -int(self.level_id)
+        except TypeError as e:
+            logger.error(
+                "level_id: {self.level_id} latest_time: {self.latest_time}",
+                exc_info=e)
         super(IastVulnerabilityModel, self).save(*args, **kwargs)
 
 

From b6687a3cf6be90445d9bb5e310313a6eec5eea6c Mon Sep 17 00:00:00 2001
From: st1020 
Date: Mon, 17 Jul 2023 16:22:40 +0800
Subject: [PATCH 155/161] fix: fix IastVulRecheckPayloadSerializer error

---
 dongtai_web/vul_recheck_payload/vul_recheck_payload.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dongtai_web/vul_recheck_payload/vul_recheck_payload.py b/dongtai_web/vul_recheck_payload/vul_recheck_payload.py
index 0ccc9597b..eaeac21c7 100644
--- a/dongtai_web/vul_recheck_payload/vul_recheck_payload.py
+++ b/dongtai_web/vul_recheck_payload/vul_recheck_payload.py
@@ -19,7 +19,7 @@ def get_or_none(classmodel, function, **kwargs):
 
 class IastVulRecheckPayloadSerializer(serializers.Serializer):
     value = serializers.CharField()
-    status = serializers.IntegerField()
+    status = serializers.IntegerField(min_value=1, max_value=2**31 - 1)
     strategy_id = serializers.IntegerField(min_value=1, max_value=2**31 - 1)
     language_id = serializers.IntegerField(min_value=1, max_value=2**31 - 1)
 

From 7799efe0167b1a0380880d61801231850e867a0a Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 17 Jul 2023 08:28:23 +0000
Subject: [PATCH 156/161] build(deps): bump cryptography from 41.0.0 to 41.0.2

Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.0 to 41.0.2.
- [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/pyca/cryptography/compare/41.0.0...41.0.2)

---
updated-dependencies:
- dependency-name: cryptography
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index ff4536c5c..45d971cc5 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -24,7 +24,7 @@ click-didyoumean==0.3.0 ; python_full_version >= '3.6.2' and python_full_version
 click-plugins==1.1.1
 click-repl==0.3.0 ; python_version >= '3.6'
 crcmod==1.7
-cryptography==41.0.0
+cryptography==41.0.2
 dataclasses-json==0.5.9
 ddt==1.6.0
 defusedxml==0.7.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'

From d6d8054f00eb4d8d89fe5cb02b5f1ff833514197 Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Mon, 17 Jul 2023 16:53:16 +0800
Subject: [PATCH 157/161] fix: source method repeated.

---
 dongtai_common/engine/vul_engine.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dongtai_common/engine/vul_engine.py b/dongtai_common/engine/vul_engine.py
index 738cad93c..ee5afb4b4 100644
--- a/dongtai_common/engine/vul_engine.py
+++ b/dongtai_common/engine/vul_engine.py
@@ -184,7 +184,7 @@ def search(self, method_pool, vul_method_signature, vul_type=None):
                         self.vul_source_signature = f"{sub_method.get('className')}.{sub_method.get('methodName')}"
                         final_stack.append(
                             self.copy_method(sub_method, source=True))
-                    if sub_method['invokeId'] == t:
+                    elif sub_method['invokeId'] == t:
                         self.taint_value = sub_method['targetValues']
                         final_stack.append(
                             self.copy_method(sub_method, sink=True))

From a8f9b25861cf280ce207c548188c221fe1d15a46 Mon Sep 17 00:00:00 2001
From: tscuite 
Date: Mon, 17 Jul 2023 17:30:31 +0800
Subject: [PATCH 158/161] feat: add ci

---
 deploy/kubernetes/helm/templates/_helpers.tpl              | 2 +-
 .../helm/templates/configmaps/dongtai-logstash.yaml        | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/deploy/kubernetes/helm/templates/_helpers.tpl b/deploy/kubernetes/helm/templates/_helpers.tpl
index 477a9ae98..f7d318144 100644
--- a/deploy/kubernetes/helm/templates/_helpers.tpl
+++ b/deploy/kubernetes/helm/templates/_helpers.tpl
@@ -135,7 +135,7 @@ imagePullPolicy: {{.Values.imagePullPolicy}}
 {{- end -}}
 {{- define "deploy.devinitContainers" -}}
 initContainers:
-  - name: init-dongtai-agentcontainer
+  - name: init-dongtai-agentcontainercache
     command: ["/bin/sh", "-c"]
     args: ["cd /tmp/iast_cache/package && curl -s https://charts.dongtai.io/agent_{{.Values.develop.agentZip}}/java/latest/agent_latest.tar.gz | tar -xvzf -"]
     image: curlimages/curl
diff --git a/deploy/kubernetes/helm/templates/configmaps/dongtai-logstash.yaml b/deploy/kubernetes/helm/templates/configmaps/dongtai-logstash.yaml
index 8ea794012..6c9519288 100644
--- a/deploy/kubernetes/helm/templates/configmaps/dongtai-logstash.yaml
+++ b/deploy/kubernetes/helm/templates/configmaps/dongtai-logstash.yaml
@@ -37,6 +37,13 @@ data:
         # }
       }
       if [type] == "log"{
+        # ruby {
+        #   init => "require 'base64'"
+        #   code => "event.set('message', Base64.decode64(event.get('message')))"
+        # }
+        # ruby {
+        #   code => "puts event.to_hash"
+        # }
         json{
           source => ["message"]
           remove_field => ["message"]

From 0c80ce05ef766f38b51280a8edb16c7b8adb5207 Mon Sep 17 00:00:00 2001
From: bidaya0 
Date: Mon, 17 Jul 2023 18:01:44 +0800
Subject: [PATCH 159/161] fix/logstash_parse_json_error

---
 dongtai_protocol/report/log_service.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/dongtai_protocol/report/log_service.py b/dongtai_protocol/report/log_service.py
index 58d9ebaa8..052a61373 100644
--- a/dongtai_protocol/report/log_service.py
+++ b/dongtai_protocol/report/log_service.py
@@ -1,6 +1,5 @@
 import logging
 import socket
-from base64 import b64encode
 
 logger = logging.getLogger('dongtai.openapi')
 
@@ -39,9 +38,8 @@ def send(self, message):
             if not self.socket:
                 self.create_socket()
             if self.socket:
-                self.socket.sendall(
-                    b64encode(bytes(message, encoding='utf-8')) +
-                    bytes("\n", encoding='utf-8'), socket.MSG_DONTWAIT)
+                self.socket.sendall(bytes(message + "\n", encoding='utf-8'),
+                                    socket.MSG_DONTWAIT)
                 return True
         except Exception as e:
             logger.error('failed to send message to log service', exc_info=e)

From 073c03e73a0679ea129e0b89bb857234bd70fb09 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Mon, 17 Jul 2023 18:33:15 +0800
Subject: [PATCH 160/161] docs: fix schema error

---
 dongtai_web/views/details_id.py        | 5 +++--
 dongtai_web/views/new_project_query.py | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/dongtai_web/views/details_id.py b/dongtai_web/views/details_id.py
index a668d800c..bc05689a4 100644
--- a/dongtai_web/views/details_id.py
+++ b/dongtai_web/views/details_id.py
@@ -90,7 +90,8 @@ class ProjectListWithid(DetailListWithid):
 
     @extend_schema(
         tags=[_('Project')],
-        summary=_('Project List with id'),
+        summary=_('通过ID获取项目列表'),
+        description=_('通过ID获取项目列表'),
     )
     def get(self, request):
         return super().get(request)
@@ -104,7 +105,7 @@ def query(self, ids, request):
     @extend_schema_with_envcheck(
         request=IdsSerializer,
         tags=[_('Project')],
-        summary=_('Project List with id'),
+        summary=_('通过ID获取项目列表'),
         description=_("Get the item corresponding to the user, support fuzzy search based on name."
                       ),
     )
diff --git a/dongtai_web/views/new_project_query.py b/dongtai_web/views/new_project_query.py
index a1dfbe38b..1b69cd195 100644
--- a/dongtai_web/views/new_project_query.py
+++ b/dongtai_web/views/new_project_query.py
@@ -37,7 +37,7 @@ class NewProjectVersionList(UserEndPoint):
     @extend_schema_with_envcheck(
         [ProjectVersionArgSerializer],
         tags=[_('Project')],
-        summary=_('Projects List'),
+        summary=_('项目版本列表'),
         description=_("Get the item corresponding to the user, support fuzzy search based on name."),
     )
     def get(self, request):

From e6a164a7b0c1ceeb7855e99bcbfbc82cd4c748b4 Mon Sep 17 00:00:00 2001
From: st1020 
Date: Mon, 17 Jul 2023 18:44:06 +0800
Subject: [PATCH 161/161] docs: fix schema error

---
 dongtai_common/models/project.py | 6 +++---
 dongtai_web/views/projects.py    | 9 ++++-----
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/dongtai_common/models/project.py b/dongtai_common/models/project.py
index 942f7a1a1..c8299f37a 100644
--- a/dongtai_common/models/project.py
+++ b/dongtai_common/models/project.py
@@ -21,9 +21,9 @@ class VulValidation(models.IntegerChoices):
 
 
 class ProjectStatus(models.IntegerChoices):
-    NORMAL = 0
-    ERROR = 1
-    OFFLINE = 2
+    NORMAL = 0, "正常"
+    ERROR = 1, "错误"
+    OFFLINE = 2, "离线"
     __empty__ = 0
 
 class IastProjectTemplate(models.Model):
diff --git a/dongtai_web/views/projects.py b/dongtai_web/views/projects.py
index e9af2475a..e38ba5bd6 100644
--- a/dongtai_web/views/projects.py
+++ b/dongtai_web/views/projects.py
@@ -8,7 +8,7 @@
 
 from dongtai_common.endpoint import R
 from dongtai_common.endpoint import UserEndPoint
-from dongtai_common.models.project import IastProject
+from dongtai_common.models.project import IastProject, ProjectStatus
 from dongtai_web.serializers.project import (
     ProjectSerializer,
     get_vul_levels_dict,
@@ -30,12 +30,11 @@ class _ProjectsArgsSerializer(serializers.Serializer):
         default=None,
         help_text=_("The name of the item to be searched, supports fuzzy search."),
     )
-    status = serializers.IntegerField(
+    status = serializers.ChoiceField(
+        ProjectStatus.choices,
         default=None,
         allow_null=True,
-        min_value=0,
-        max_value=10,
-        help_text=_("The project status."),
+        help_text="".join([f" {i.label}: {i.value} " for i in ProjectStatus]),
     )
     exclude_vul_status = serializers.IntegerField(
         default=None,