From b3a889cc7891fbf5082232691de9d1d192f44eca Mon Sep 17 00:00:00 2001 From: Babic Date: Tue, 21 Feb 2023 08:48:02 +0100 Subject: [PATCH 1/7] Aake sql query out of code. Set the same columns in account_map when creating dummy and organization account_map --- .../data/queries/shared/account_map_dummy.sql | 12 ++++++++---- .../core/data/queries/shared/account_map_org.sql | 6 ++++++ cid/helpers/account_map.py | 16 ++++++---------- 3 files changed, 20 insertions(+), 14 deletions(-) create mode 100644 cid/builtin/core/data/queries/shared/account_map_org.sql diff --git a/cid/builtin/core/data/queries/shared/account_map_dummy.sql b/cid/builtin/core/data/queries/shared/account_map_dummy.sql index 54e62769..bc163041 100644 --- a/cid/builtin/core/data/queries/shared/account_map_dummy.sql +++ b/cid/builtin/core/data/queries/shared/account_map_dummy.sql @@ -1,10 +1,14 @@ -CREATE OR REPLACE VIEW ${athena_view_name} AS +CREATE OR REPLACE VIEW ${athena_view_name} AS SELECT DISTINCT line_item_usage_account_id account_id, - MAX_BY(bill_payer_account_id, line_item_usage_start_date) parent_account_id, MAX_BY(line_item_usage_account_id, line_item_usage_start_date) account_name, - MAX_BY(line_item_usage_account_id, line_item_usage_start_date) account_email_id + MAX_BY(bill_payer_account_id, line_item_usage_start_date) parent_account_id, + CAST('' AS varchar) account_status, + CAST('' AS varchar) account_email, + CAST('' AS varchar) account_email_id FROM "${cur_table_name}" +WHERE + line_item_usage_account_id <> '' GROUP BY - line_item_usage_account_id + line_item_usage_account_id \ No newline at end of file diff --git a/cid/builtin/core/data/queries/shared/account_map_org.sql b/cid/builtin/core/data/queries/shared/account_map_org.sql new file mode 100644 index 00000000..3be2f8bc --- /dev/null +++ b/cid/builtin/core/data/queries/shared/account_map_org.sql @@ -0,0 +1,6 @@ +CREATE OR REPLACE VIEW ${athena_view_name} AS +SELECT + * +FROM + ( VALUES ${rows} ) +ignored_table_name (account_id, account_name, parent_account_id, account_status, account_email, account_email_id) \ No newline at end of file diff --git a/cid/helpers/account_map.py b/cid/helpers/account_map.py index 1dd9b336..8065b953 100644 --- a/cid/helpers/account_map.py +++ b/cid/helpers/account_map.py @@ -88,7 +88,6 @@ def accounts(self) -> dict: def create(self, name) -> bool: """Create account map""" - print(f'\nCreating {name}...') logger.info(f'Creating account mapping "{name}"...') try: @@ -264,20 +263,17 @@ def create_account_mapping_sql(self, name) -> str: if self._metadata_source == 'dummy': return self.get_dummy_account_mapping_sql(name) - template_str = '''CREATE OR REPLACE VIEW ${athena_view_name} AS - SELECT - * - FROM - ( VALUES ${rows} ) - ignored_table_name (account_id, account_name, parent_account_id, account_status, account_email) - ''' - template = Template(template_str) + template = Template(resource_string( + package_or_requirement='cid.builtin.core', + resource_name='data/queries/shared/account_map_org.sql', + ).decode('utf-8')) + accounts_sql = list() for account in self.accounts: acc = account.copy() account_name = acc.pop('account_name').replace("'", "''") accounts_sql.append( - """ROW ('{account_id}', '{account_name}:{account_id}', '{parent_account_id}', '{account_status}', '{account_email}')""".format(account_name=account_name, **acc)) + """ROW ('{account_id}', '{account_name}:{account_id}', '{parent_account_id}', '{account_status}', '{account_email}', '{account_email}')""".format(account_name=account_name, **acc)) # Fill in TPLs columns_tpl = { 'athena_view_name': name, From 37c41ec552df75aa9bb0f982b4a394fa60d00695 Mon Sep 17 00:00:00 2001 From: Babic Date: Tue, 21 Feb 2023 10:05:31 +0100 Subject: [PATCH 2/7] create account_map view as from template instead from python code --- cfn-templates/cudos-cfn.yml | 56 +++++++++++++------ cfn-templates/views_cid.json | 7 +++ .../data/queries/shared/account_map_cfn.sql | 14 +++++ 3 files changed, 61 insertions(+), 16 deletions(-) create mode 100644 cid/builtin/core/data/queries/shared/account_map_cfn.sql diff --git a/cfn-templates/cudos-cfn.yml b/cfn-templates/cudos-cfn.yml index 608a7e76..406a8933 100644 --- a/cfn-templates/cudos-cfn.yml +++ b/cfn-templates/cudos-cfn.yml @@ -295,22 +295,6 @@ Resources: WorkGroup=athena_workgroup ) logger.info('Completed step %s',step) - # Create a dummy account map. Can be modified later - account_map_query = f""" - CREATE OR REPLACE VIEW account_map - AS SELECT DISTINCT - line_item_usage_account_id account_id, - line_item_usage_account_id account_name - FROM \"{table_name}\" - """ - step = f"Executing account map query: {repr(account_map_query)}" - logger.info(step) - response = athena.start_query_execution( - QueryString=account_map_query, - QueryExecutionContext={'Database': database_name}, - WorkGroup=athena_workgroup, - ) - logger.info("Completed %s", step) return(True, "Views created successfully") except Exception as exc: logger.error(exc, exc_info=True) @@ -1419,6 +1403,14 @@ Resources: Type: STRING - Name: account_name Type: STRING + - Name: parent_account_id + Type: STRING + - Name: account_status + Type: STRING + - Name: account_email_id + Type: STRING + - Name: account_email + Type: STRING Name: account_map Schema: !Ref CURDatabaseName Catalog: AwsDataCatalog @@ -1506,6 +1498,14 @@ Resources: Type: STRING - Name: account_name Type: STRING + - Name: parent_account_id + Type: STRING + - Name: account_status + Type: STRING + - Name: account_email_id + Type: STRING + - Name: account_email + Type: STRING Name: account_map Schema: !Ref CURDatabaseName Permissions: @@ -1621,6 +1621,14 @@ Resources: Type: STRING - Name: account_name Type: STRING + - Name: parent_account_id + Type: STRING + - Name: account_status + Type: STRING + - Name: account_email_id + Type: STRING + - Name: account_email + Type: STRING Name: account_map Schema: !Ref CURDatabaseName Catalog: AwsDataCatalog @@ -1690,6 +1698,14 @@ Resources: Type: STRING - Name: account_name Type: STRING + - Name: parent_account_id + Type: STRING + - Name: account_status + Type: STRING + - Name: account_email_id + Type: STRING + - Name: account_email + Type: STRING Name: account_map Schema: !Ref CURDatabaseName ecb79e5e-1428-4f43-80bc-e547a93da21a: @@ -1837,6 +1853,14 @@ Resources: Type: STRING - Name: account_name Type: STRING + - Name: parent_account_id + Type: STRING + - Name: account_status + Type: STRING + - Name: account_email_id + Type: STRING + - Name: account_email + Type: STRING Name: account_map Schema: !Ref CURDatabaseName Catalog: AwsDataCatalog diff --git a/cfn-templates/views_cid.json b/cfn-templates/views_cid.json index a9c94418..7bb7812f 100644 --- a/cfn-templates/views_cid.json +++ b/cfn-templates/views_cid.json @@ -35,6 +35,13 @@ "spFile": "cid/summary_view_sp.sql", "riFile": "cid/summary_view_ri.sql", "File": "cid/summary_view.sql" + }, + "account_map": { + "__description__": "", + "spriFile": "shared/account_map_cfn.sql", + "spFile": "shared/account_map_cfn.sql", + "riFile": "shared/account_map_cfn.sql", + "File": "shared/account_map_cfn.sql" } } } \ No newline at end of file diff --git a/cid/builtin/core/data/queries/shared/account_map_cfn.sql b/cid/builtin/core/data/queries/shared/account_map_cfn.sql new file mode 100644 index 00000000..ff376c2b --- /dev/null +++ b/cid/builtin/core/data/queries/shared/account_map_cfn.sql @@ -0,0 +1,14 @@ +CREATE OR REPLACE VIEW account_map AS +SELECT DISTINCT + line_item_usage_account_id account_id, + MAX_BY(line_item_usage_account_id, line_item_usage_start_date) account_name, + MAX_BY(bill_payer_account_id, line_item_usage_start_date) parent_account_id, + CAST('' AS varchar) account_status, + CAST('' AS varchar) account_email, + CAST('' AS varchar) account_email_id +FROM + "${cur_table_name}" +WHERE + line_item_usage_account_id <> '' +GROUP BY + line_item_usage_account_id \ No newline at end of file From a7ed5049842a870d6813835372ad242d73308abe Mon Sep 17 00:00:00 2001 From: Babic Date: Tue, 21 Feb 2023 10:12:54 +0100 Subject: [PATCH 3/7] create account_map table from template, not from code --- cfn-templates/kpi.cfn.yml | 12 ++++++++---- cfn-templates/views_kpi.json | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/cfn-templates/kpi.cfn.yml b/cfn-templates/kpi.cfn.yml index 84b43c6e..b7fd7ee0 100644 --- a/cfn-templates/kpi.cfn.yml +++ b/cfn-templates/kpi.cfn.yml @@ -477,10 +477,6 @@ Resources: Properties: ServiceToken: !GetAtt KPICustomResourceViewsLambdaHandler.Arn ViewName: !FindInMap [ "Definitions", "sharedAccountMapView", "Value" ] - ViewCode: !Join - - "" - - - "CREATE OR REPLACE VIEW account_map AS SELECT DISTINCT line_item_usage_account_id account_id, line_item_usage_account_id account_name FROM " - - !Ref CURTableName TableName: !Ref CURTableName # ----- sharedSummaryViewCustom: @@ -1640,6 +1636,14 @@ Resources: Type: STRING - Name: account_name Type: STRING + - Name: parent_account_id + Type: STRING + - Name: account_status + Type: STRING + - Name: account_email_id + Type: STRING + - Name: account_email + Type: STRING Name: account_map Schema: !Ref CURDatabaseName Catalog: AwsDataCatalog diff --git a/cfn-templates/views_kpi.json b/cfn-templates/views_kpi.json index 11a4ee38..abd4b485 100644 --- a/cfn-templates/views_kpi.json +++ b/cfn-templates/views_kpi.json @@ -36,7 +36,7 @@ "account_map": { "__description__": "", "Path": "https://raw.githubusercontent.com/aws-samples/aws-cudos-framework-deployment/main/cid/builtin/core/data/queries/shared", - "File": "account_map.sql" + "File": "account_map_cfn.sql" }, "summary_view": { "__description__": "", From 161a4c4eb976cda5a27912fc2abeb5a5d871c658 Mon Sep 17 00:00:00 2001 From: Babic Date: Wed, 1 Mar 2023 12:06:35 +0100 Subject: [PATCH 4/7] Revert "create account_map view as from template instead from python code" This reverts commit 37c41ec552df75aa9bb0f982b4a394fa60d00695. --- cfn-templates/cudos-cfn.yml | 56 ++++++------------- cfn-templates/views_cid.json | 7 --- .../data/queries/shared/account_map_cfn.sql | 14 ----- 3 files changed, 16 insertions(+), 61 deletions(-) delete mode 100644 cid/builtin/core/data/queries/shared/account_map_cfn.sql diff --git a/cfn-templates/cudos-cfn.yml b/cfn-templates/cudos-cfn.yml index 406a8933..608a7e76 100644 --- a/cfn-templates/cudos-cfn.yml +++ b/cfn-templates/cudos-cfn.yml @@ -295,6 +295,22 @@ Resources: WorkGroup=athena_workgroup ) logger.info('Completed step %s',step) + # Create a dummy account map. Can be modified later + account_map_query = f""" + CREATE OR REPLACE VIEW account_map + AS SELECT DISTINCT + line_item_usage_account_id account_id, + line_item_usage_account_id account_name + FROM \"{table_name}\" + """ + step = f"Executing account map query: {repr(account_map_query)}" + logger.info(step) + response = athena.start_query_execution( + QueryString=account_map_query, + QueryExecutionContext={'Database': database_name}, + WorkGroup=athena_workgroup, + ) + logger.info("Completed %s", step) return(True, "Views created successfully") except Exception as exc: logger.error(exc, exc_info=True) @@ -1403,14 +1419,6 @@ Resources: Type: STRING - Name: account_name Type: STRING - - Name: parent_account_id - Type: STRING - - Name: account_status - Type: STRING - - Name: account_email_id - Type: STRING - - Name: account_email - Type: STRING Name: account_map Schema: !Ref CURDatabaseName Catalog: AwsDataCatalog @@ -1498,14 +1506,6 @@ Resources: Type: STRING - Name: account_name Type: STRING - - Name: parent_account_id - Type: STRING - - Name: account_status - Type: STRING - - Name: account_email_id - Type: STRING - - Name: account_email - Type: STRING Name: account_map Schema: !Ref CURDatabaseName Permissions: @@ -1621,14 +1621,6 @@ Resources: Type: STRING - Name: account_name Type: STRING - - Name: parent_account_id - Type: STRING - - Name: account_status - Type: STRING - - Name: account_email_id - Type: STRING - - Name: account_email - Type: STRING Name: account_map Schema: !Ref CURDatabaseName Catalog: AwsDataCatalog @@ -1698,14 +1690,6 @@ Resources: Type: STRING - Name: account_name Type: STRING - - Name: parent_account_id - Type: STRING - - Name: account_status - Type: STRING - - Name: account_email_id - Type: STRING - - Name: account_email - Type: STRING Name: account_map Schema: !Ref CURDatabaseName ecb79e5e-1428-4f43-80bc-e547a93da21a: @@ -1853,14 +1837,6 @@ Resources: Type: STRING - Name: account_name Type: STRING - - Name: parent_account_id - Type: STRING - - Name: account_status - Type: STRING - - Name: account_email_id - Type: STRING - - Name: account_email - Type: STRING Name: account_map Schema: !Ref CURDatabaseName Catalog: AwsDataCatalog diff --git a/cfn-templates/views_cid.json b/cfn-templates/views_cid.json index 7bb7812f..a9c94418 100644 --- a/cfn-templates/views_cid.json +++ b/cfn-templates/views_cid.json @@ -35,13 +35,6 @@ "spFile": "cid/summary_view_sp.sql", "riFile": "cid/summary_view_ri.sql", "File": "cid/summary_view.sql" - }, - "account_map": { - "__description__": "", - "spriFile": "shared/account_map_cfn.sql", - "spFile": "shared/account_map_cfn.sql", - "riFile": "shared/account_map_cfn.sql", - "File": "shared/account_map_cfn.sql" } } } \ No newline at end of file diff --git a/cid/builtin/core/data/queries/shared/account_map_cfn.sql b/cid/builtin/core/data/queries/shared/account_map_cfn.sql deleted file mode 100644 index ff376c2b..00000000 --- a/cid/builtin/core/data/queries/shared/account_map_cfn.sql +++ /dev/null @@ -1,14 +0,0 @@ -CREATE OR REPLACE VIEW account_map AS -SELECT DISTINCT - line_item_usage_account_id account_id, - MAX_BY(line_item_usage_account_id, line_item_usage_start_date) account_name, - MAX_BY(bill_payer_account_id, line_item_usage_start_date) parent_account_id, - CAST('' AS varchar) account_status, - CAST('' AS varchar) account_email, - CAST('' AS varchar) account_email_id -FROM - "${cur_table_name}" -WHERE - line_item_usage_account_id <> '' -GROUP BY - line_item_usage_account_id \ No newline at end of file From b00c1577edf7255843ad27a46d7009144477bfed Mon Sep 17 00:00:00 2001 From: Babic Date: Wed, 1 Mar 2023 12:39:13 +0100 Subject: [PATCH 5/7] revert changes as those files are not used any more --- cfn-templates/kpi.cfn.yml | 12 ++++-------- cfn-templates/views_kpi.json | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/cfn-templates/kpi.cfn.yml b/cfn-templates/kpi.cfn.yml index b7fd7ee0..84b43c6e 100644 --- a/cfn-templates/kpi.cfn.yml +++ b/cfn-templates/kpi.cfn.yml @@ -477,6 +477,10 @@ Resources: Properties: ServiceToken: !GetAtt KPICustomResourceViewsLambdaHandler.Arn ViewName: !FindInMap [ "Definitions", "sharedAccountMapView", "Value" ] + ViewCode: !Join + - "" + - - "CREATE OR REPLACE VIEW account_map AS SELECT DISTINCT line_item_usage_account_id account_id, line_item_usage_account_id account_name FROM " + - !Ref CURTableName TableName: !Ref CURTableName # ----- sharedSummaryViewCustom: @@ -1636,14 +1640,6 @@ Resources: Type: STRING - Name: account_name Type: STRING - - Name: parent_account_id - Type: STRING - - Name: account_status - Type: STRING - - Name: account_email_id - Type: STRING - - Name: account_email - Type: STRING Name: account_map Schema: !Ref CURDatabaseName Catalog: AwsDataCatalog diff --git a/cfn-templates/views_kpi.json b/cfn-templates/views_kpi.json index abd4b485..11a4ee38 100644 --- a/cfn-templates/views_kpi.json +++ b/cfn-templates/views_kpi.json @@ -36,7 +36,7 @@ "account_map": { "__description__": "", "Path": "https://raw.githubusercontent.com/aws-samples/aws-cudos-framework-deployment/main/cid/builtin/core/data/queries/shared", - "File": "account_map_cfn.sql" + "File": "account_map.sql" }, "summary_view": { "__description__": "", From ee6699a896ebbba2d0c10bd8307e8c515a0f0963 Mon Sep 17 00:00:00 2001 From: Babic Date: Wed, 1 Mar 2023 12:49:10 +0100 Subject: [PATCH 6/7] removed account_email_id column in account_map --- cid/builtin/core/data/queries/shared/account_map_dummy.sql | 3 +-- cid/builtin/core/data/queries/shared/account_map_org.sql | 2 +- cid/helpers/account_map.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cid/builtin/core/data/queries/shared/account_map_dummy.sql b/cid/builtin/core/data/queries/shared/account_map_dummy.sql index bc163041..7943a8b8 100644 --- a/cid/builtin/core/data/queries/shared/account_map_dummy.sql +++ b/cid/builtin/core/data/queries/shared/account_map_dummy.sql @@ -4,8 +4,7 @@ SELECT DISTINCT MAX_BY(line_item_usage_account_id, line_item_usage_start_date) account_name, MAX_BY(bill_payer_account_id, line_item_usage_start_date) parent_account_id, CAST('' AS varchar) account_status, - CAST('' AS varchar) account_email, - CAST('' AS varchar) account_email_id + CAST('' AS varchar) account_email FROM "${cur_table_name}" WHERE diff --git a/cid/builtin/core/data/queries/shared/account_map_org.sql b/cid/builtin/core/data/queries/shared/account_map_org.sql index 3be2f8bc..38fa07d5 100644 --- a/cid/builtin/core/data/queries/shared/account_map_org.sql +++ b/cid/builtin/core/data/queries/shared/account_map_org.sql @@ -3,4 +3,4 @@ SELECT * FROM ( VALUES ${rows} ) -ignored_table_name (account_id, account_name, parent_account_id, account_status, account_email, account_email_id) \ No newline at end of file +ignored_table_name (account_id, account_name, parent_account_id, account_status, account_email) \ No newline at end of file diff --git a/cid/helpers/account_map.py b/cid/helpers/account_map.py index 8065b953..8eacd528 100644 --- a/cid/helpers/account_map.py +++ b/cid/helpers/account_map.py @@ -273,7 +273,7 @@ def create_account_mapping_sql(self, name) -> str: acc = account.copy() account_name = acc.pop('account_name').replace("'", "''") accounts_sql.append( - """ROW ('{account_id}', '{account_name}:{account_id}', '{parent_account_id}', '{account_status}', '{account_email}', '{account_email}')""".format(account_name=account_name, **acc)) + """ROW ('{account_id}', '{account_name}:{account_id}', '{parent_account_id}', '{account_status}', '{account_email}')""".format(account_name=account_name, **acc)) # Fill in TPLs columns_tpl = { 'athena_view_name': name, From bd2d7e5de9b55443fada1946720dd7a1453a8e82 Mon Sep 17 00:00:00 2001 From: Babic Date: Tue, 21 Mar 2023 08:58:00 +0100 Subject: [PATCH 7/7] * removed email column from account_map view * added missing closing bracket in query in file account_map.sql --- .../core/data/queries/shared/account_map.sql | 10 +++--- .../data/queries/shared/account_map_dummy.sql | 3 +- .../data/queries/shared/account_map_org.sql | 2 +- .../core/data/queries/shared/aws_accounts.sql | 4 +-- cid/helpers/account_map.py | 31 ++++++++++--------- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/cid/builtin/core/data/queries/shared/account_map.sql b/cid/builtin/core/data/queries/shared/account_map.sql index eb5c4dfc..a6b96f2b 100644 --- a/cid/builtin/core/data/queries/shared/account_map.sql +++ b/cid/builtin/core/data/queries/shared/account_map.sql @@ -1,10 +1,12 @@ CREATE OR REPLACE VIEW account_map AS SELECT ${account_id} AS account_id, - MAX_BY(${account_name}, concat(${account_name}, ': ', ${account_id}) AS account_name -FROM + MAX_BY(${account_name}, concat(${account_name}, ': ', ${account_id})) AS account_name, + ${account_id} AS parent_account_id, + CAST('' AS varchar) account_status +FROM ${metadata_table_name} -WHERE +WHERE ${account_name} <> '' -GROUP BY +GROUP BY ${account_id} diff --git a/cid/builtin/core/data/queries/shared/account_map_dummy.sql b/cid/builtin/core/data/queries/shared/account_map_dummy.sql index 7943a8b8..8a7d10dc 100644 --- a/cid/builtin/core/data/queries/shared/account_map_dummy.sql +++ b/cid/builtin/core/data/queries/shared/account_map_dummy.sql @@ -3,8 +3,7 @@ SELECT DISTINCT line_item_usage_account_id account_id, MAX_BY(line_item_usage_account_id, line_item_usage_start_date) account_name, MAX_BY(bill_payer_account_id, line_item_usage_start_date) parent_account_id, - CAST('' AS varchar) account_status, - CAST('' AS varchar) account_email + CAST('' AS varchar) account_status FROM "${cur_table_name}" WHERE diff --git a/cid/builtin/core/data/queries/shared/account_map_org.sql b/cid/builtin/core/data/queries/shared/account_map_org.sql index 38fa07d5..cbd7e6ff 100644 --- a/cid/builtin/core/data/queries/shared/account_map_org.sql +++ b/cid/builtin/core/data/queries/shared/account_map_org.sql @@ -3,4 +3,4 @@ SELECT * FROM ( VALUES ${rows} ) -ignored_table_name (account_id, account_name, parent_account_id, account_status, account_email) \ No newline at end of file +ignored_table_name (account_id, account_name, parent_account_id, account_status) \ No newline at end of file diff --git a/cid/builtin/core/data/queries/shared/aws_accounts.sql b/cid/builtin/core/data/queries/shared/aws_accounts.sql index 0511c7c3..90e2fd2a 100644 --- a/cid/builtin/core/data/queries/shared/aws_accounts.sql +++ b/cid/builtin/core/data/queries/shared/aws_accounts.sql @@ -1,8 +1,7 @@ CREATE OR REPLACE VIEW aws_accounts AS WITH m AS ( SELECT ${account_id} as account_id, - ${account_name} as account_name, - email account_email_id + ${account_name} as account_name FROM ${metadata_table_name} ), cur AS ( @@ -13,7 +12,6 @@ CREATE OR REPLACE VIEW aws_accounts AS WITH SELECT m.account_id, m.account_name, cur.parent_account_id, - m.account_email_id, 'Active' account_status FROM ( m diff --git a/cid/helpers/account_map.py b/cid/helpers/account_map.py index 8eacd528..b93055da 100644 --- a/cid/helpers/account_map.py +++ b/cid/helpers/account_map.py @@ -24,11 +24,11 @@ class AccountMap(CidBase): mappings = { 'account_map': { 'acc_metadata_details': {'account_id': 'account_id', 'account_name': 'account_name'}, - 'organisation_data': {'account_id': 'id', 'account_name': 'name', 'email': 'email'} + 'organisation_data': {'account_id': 'id', 'account_name': 'name'} }, 'aws_accounts': { - 'acc_metadata_details': {'account_id': 'account_id', 'account_name': 'account_name', 'email': 'email'}, - 'organisation_data': { 'account_id': 'id', 'account_name': 'name', 'email': 'email', 'status': 'status'}, + 'acc_metadata_details': {'account_id': 'account_id', 'account_name': 'account_name'}, + 'organisation_data': { 'account_id': 'id', 'account_name': 'name', 'status': 'status'}, 'cur_fields': ['bill_payer_account_id'] } } @@ -82,7 +82,6 @@ def accounts(self) -> dict: account.update({ 'parent_account_id': account.get('parent_account_id', '0'), 'account_status': account.get('account_status', 'unknown'), - 'account_email': account.get('account_email', 'unknown') }) return self._accounts @@ -155,10 +154,7 @@ def create(self, name) -> bool: def get_dummy_account_mapping_sql(self, name) -> list: """Create dummy account mapping""" logger.info(f'Creating dummy account mapping for {name}') - template = Template(resource_string( - package_or_requirement='cid.builtin.core', - resource_name='data/queries/shared/account_map_dummy.sql', - ).decode('utf-8')) + template = self.get_template('data/queries/shared/account_map_dummy.sql') columns_tpl = { 'athena_view_name': name, 'cur_table_name': self.cur.tableName @@ -179,8 +175,7 @@ def get_organization_accounts(self) -> list: accounts.append({ 'account_id': account.get('Id'), 'account_name': account.get('Name'), - 'account_status': account.get('Status'), - 'account_email': account.get('Email') + 'account_status': account.get('Status') }) except orgs.exceptions.AWSOrganizationsNotInUseException: print('AWS Organization is not enabled') @@ -198,6 +193,15 @@ def check_file_exists(self, file_path) -> bool: return Path.is_file(abs_path / file_path) + + def get_template(self, template_name): + """ Create template from provided template file """ + return Template(resource_string( + package_or_requirement='cid.builtin.core', + resource_name=template_name, + ).decode('utf-8')) + + def get_csv_accounts(self, file_path) -> list: """ Retreive accounts from CSV file """ with open(file_path) as f: @@ -263,17 +267,14 @@ def create_account_mapping_sql(self, name) -> str: if self._metadata_source == 'dummy': return self.get_dummy_account_mapping_sql(name) - template = Template(resource_string( - package_or_requirement='cid.builtin.core', - resource_name='data/queries/shared/account_map_org.sql', - ).decode('utf-8')) + template = self.get_template('data/queries/shared/account_map_org.sql') accounts_sql = list() for account in self.accounts: acc = account.copy() account_name = acc.pop('account_name').replace("'", "''") accounts_sql.append( - """ROW ('{account_id}', '{account_name}:{account_id}', '{parent_account_id}', '{account_status}', '{account_email}')""".format(account_name=account_name, **acc)) + """ROW ('{account_id}', '{account_name}:{account_id}', '{parent_account_id}', '{account_status}')""".format(account_name=account_name, **acc)) # Fill in TPLs columns_tpl = { 'athena_view_name': name,