Skip to content

Commit

Permalink
update push-secrets and tls
Browse files Browse the repository at this point in the history
  • Loading branch information
@dghelm
dghelm committed Oct 4, 2024
1 parent 58a4d9d commit 2020121
Show file tree
Hide file tree
Showing 3 changed files with 155 additions and 86 deletions.
64 changes: 45 additions & 19 deletions src/commands/setup/configs.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,23 +112,25 @@ export default class SetupConfigs extends Command {
const config = toml.parse(configContent)

const services = [
'blockscout', 'bridge-history-api', 'bridge-history-fetcher', 'chain-monitor', 'coordinator-cron', 'coordinator-api',
'admin-system-backend', 'blockscout', 'bridge-history-api', 'bridge-history-fetcher', 'chain-monitor', 'coordinator-api', 'coordinator-cron',
'gas-oracle', 'l1-explorer', 'l2-sequencer', 'rollup-node'
]

for (const service of services) {
const envFile = path.join(process.cwd(), 'secrets', `${service}-secret.env`)
const envContent = this.generateEnvContent(service, config)
fs.writeFileSync(envFile, envContent)
this.log(chalk.green(`Created ${service}-secret.env`))
const envFiles = this.generateEnvContent(service, config)
for (const [filename, content] of Object.entries(envFiles)) {
const envFile = path.join(process.cwd(), 'secrets', filename)
fs.writeFileSync(envFile, content)
this.log(chalk.green(`Created ${filename}`))
}
}

// Create additional files
this.createMigrateDbFiles(config)
}

// TODO: check privatekey secrets once integrated
private generateEnvContent(service: string, config: any): string {
private generateEnvContent(service: string, config: any): { [key: string]: string } {
const mapping: Record<string, string[]> = {
'admin-system-backend': ['ADMIN_SYSTEM_BACKEND_DB_CONNECTION_STRING:SCROLL_ADMIN_AUTH_DB_CONFIG', 'ADMIN_SYSTEM_BACKEND_DB_CONNECTION_STRING:SCROLL_ADMIN_DB_CONFIG_DSN', 'ADMIN_SYSTEM_BACKEND_DB_CONNECTION_STRING:SCROLL_ADMION_READ_ONLY_DB_CONFIG_DSN'],
'blockscout': ['BLOCKSCOUT_DB_CONNECTION_STRING:DATABASE_URL'],
Expand All @@ -139,24 +141,48 @@ export default class SetupConfigs extends Command {
'coordinator-cron': ['COORDINATOR_DB_CONNECTION_STRING:SCROLL_COORDINATOR_DB_DSN', 'COORDINATOR_JWT_SECRET_KEY:SCROLL_COORDINATOR_AUTH_SECRET'],
'gas-oracle': ['GAS_ORACLE_DB_CONNECTION_STRING:SCROLL_ROLLUP_DB_CONFIG_DSN', 'L1_GAS_ORACLE_SENDER_PRIVATE_KEY:SCROLL_ROLLUP_L1_CONFIG_RELAYER_CONFIG_GAS_ORACLE_SENDER_PRIVATE_KEY', 'L2_GAS_ORACLE_SENDER_PRIVATE_KEY:SCROLL_ROLLUP_L2_CONFIG_RELAYER_CONFIG_GAS_ORACLE_SENDER_PRIVATE_KEY'],
'l1-explorer': ['L1_EXPLORER_DB_CONNECTION_STRING:DATABASE_URL'],
'l2-sequencer': ['L2GETH_KEYSTORE:L2GETH_KEYSTORE_1', 'L2GETH_PASSWORD:L2GETH_PASSWORD_1', 'L2GETH_NODEKEY:L2GETH_NODEKEY_1'],
'l2-sequencer': ['L2GETH_KEYSTORE:L2GETH_KEYSTORE', 'L2GETH_PASSWORD:L2GETH_PASSWORD', 'L2GETH_NODEKEY:L2GETH_NODEKEY'],
'rollup-node': ['ROLLUP_NODE_DB_CONNECTION_STRING:SCROLL_ROLLUP_DB_CONFIG_DSN', 'L1_COMMIT_SENDER_PRIVATE_KEY:SCROLL_ROLLUP_L2_CONFIG_RELAYER_CONFIG_COMMIT_SENDER_PRIVATE_KEY', 'L1_FINALIZE_SENDER_PRIVATE_KEY:SCROLL_ROLLUP_L2_CONFIG_RELAYER_CONFIG_FINALIZE_SENDER_PRIVATE_KEY'],
}

let content = ''
for (const pair of mapping[service] || []) {
const [configKey, envKey] = pair.split(':')
if (config.db && config.db[configKey]) {
content += `${envKey}="${config.db[configKey]}"\n`
} else if (config.accounts && config.accounts[configKey]) {
content += `${envKey}="${config.accounts[configKey]}"\n`
} else if (config.coordinator && config.coordinator[configKey]) {
content += `${envKey}="${config.coordinator[configKey]}"\n`
} else if (config.sequencer && config.sequencer[configKey]) {
content += `${envKey}="${config.sequencer[configKey]}"\n`
const envFiles: { [key: string]: string } = {};

if (service === 'l2-sequencer') {
// Handle all sequencers (primary and backups)
let sequencerIndex = 0;
while (true) {
const sequencerConfig = sequencerIndex === 0 ? config.sequencer : config.sequencer[`sequencer-${sequencerIndex}`];
if (!sequencerConfig) break;

let content = '';
for (const pair of mapping[service] || []) {
const [envKey, configKey] = pair.split(':');
if (sequencerConfig[configKey]) {
content += `${envKey}_${sequencerIndex}="${sequencerConfig[configKey]}"\n`;
}
}
envFiles[`l2-sequencer-${sequencerIndex}-secret.env`] = content;
sequencerIndex++;
}
} else {
// Handle other services
let content = '';
for (const pair of mapping[service] || []) {
const [configKey, envKey] = pair.split(':');
if (config.db && config.db[configKey]) {
content += `${envKey}="${config.db[configKey]}"\n`;
} else if (config.accounts && config.accounts[configKey]) {
content += `${envKey}="${config.accounts[configKey]}"\n`;
} else if (config.coordinator && config.coordinator[configKey]) {
content += `${envKey}="${config.coordinator[configKey]}"\n`;
} else if (config.sequencer && config.sequencer[configKey]) {
content += `${envKey}="${config.sequencer[configKey]}"\n`;
}
}
envFiles[`${service}-secret.env`] = content;
}
return content

return envFiles;
}

private createMigrateDbFiles(config: any): void {
Expand Down
160 changes: 98 additions & 62 deletions src/commands/setup/push-secrets.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,26 @@ class AWSSecretService implements SecretService {

// Process ENV files
const envFiles = fs.readdirSync(secretsDir).filter(file => file.endsWith('.env'))
let l2SequencerSecrets: Record<string, string> = {}

for (const file of envFiles) {
const secretName = `${path.basename(file, '.env')}-env`
console.log(chalk.cyan(`Processing ENV secret: ${secretName}`))
const content = await this.convertToJson(path.join(secretsDir, file))
await this.pushToAWSSecret(content, secretName)
const secretName = path.basename(file, '.env')
if (secretName.startsWith('l2-sequencer-')) {
console.log(chalk.cyan(`Processing L2 Sequencer secret: ${secretName}`))
const content = await this.convertToJson(path.join(secretsDir, file))
l2SequencerSecrets = { ...l2SequencerSecrets, ...JSON.parse(content) }
} else {
console.log(chalk.cyan(`Processing ENV secret: ${secretName}`))
const content = await this.convertToJson(path.join(secretsDir, file))
await this.pushToAWSSecret(content, secretName)
}
}

// Push combined L2 Sequencer secrets
if (Object.keys(l2SequencerSecrets).length > 0) {
console.log(chalk.cyan(`Processing combined L2 Sequencer secrets: l2-sequencer-secret`))
const combinedContent = JSON.stringify(l2SequencerSecrets)
await this.pushToAWSSecret(combinedContent, 'l2-sequencer-secret')
}
}
}
Expand Down Expand Up @@ -213,11 +228,25 @@ class HashicorpVaultDevService implements SecretService {

// Process ENV files
const envFiles = fs.readdirSync(secretsDir).filter(file => file.endsWith('.env'))
let l2SequencerSecrets: Record<string, string> = {}

for (const file of envFiles) {
const secretName = `${path.basename(file, '.env')}-env`
console.log(chalk.cyan(`Processing ENV secret: scroll/${secretName}`))
const data = await this.convertEnvToDict(path.join(secretsDir, file))
await this.pushToVault(secretName, data)
const secretName = path.basename(file, '.env')
if (secretName.startsWith('l2-sequencer-')) {
console.log(chalk.cyan(`Processing L2 Sequencer secret: ${secretName}`))
const data = await this.convertEnvToDict(path.join(secretsDir, file))
l2SequencerSecrets = { ...l2SequencerSecrets, ...data }
} else {
console.log(chalk.cyan(`Processing ENV secret: scroll/${secretName}`))
const data = await this.convertEnvToDict(path.join(secretsDir, file))
await this.pushToVault(secretName, data)
}
}

// Push combined L2 Sequencer secrets
if (Object.keys(l2SequencerSecrets).length > 0) {
console.log(chalk.cyan(`Processing combined L2 Sequencer secrets: scroll/l2-sequencer-secret`))
await this.pushToVault('l2-sequencer-secret', l2SequencerSecrets)
}

console.log(chalk.green("All secrets have been processed and populated in Vault."))
Expand All @@ -230,6 +259,7 @@ export default class SetupPushSecrets extends Command {
static override examples = [
'<%= config.bin %> <%= command.id %>',
'<%= config.bin %> <%= command.id %> --debug',
'<%= config.bin %> <%= command.id %> --values-dir custom-values',
]

static override flags = {
Expand All @@ -238,8 +268,14 @@ export default class SetupPushSecrets extends Command {
description: 'Show debug output',
default: false,
}),
'values-dir': Flags.string({
description: 'Directory containing the values files',
default: 'values',
}),
}

private flags: any;

private async getVaultCredentials(): Promise<Record<string, string>> {
return {
server: await input({
Expand Down Expand Up @@ -282,94 +318,94 @@ export default class SetupPushSecrets extends Command {
}

private async updateProductionYaml(provider: string): Promise<void> {
const charts = [
'balance-checker', 'blockscout', 'blockscout-sc-verifier', 'bridge-history-api',
'bridge-history-fetcher', 'chain-monitor', 'contracts', 'coordinator-api',
'coordinator-cron', 'external-secrets-lib', 'frontends', 'gas-oracle', 'l2-bootnode', 'l2-rpc', 'l2-sequencer',
'rollup-explorer-backend', 'rollup-node', 'scroll-common', 'scroll-sdk'
]

let credentials: Record<string, string>
const valuesDir = path.join(process.cwd(), this.flags['values-dir']);
if (!fs.existsSync(valuesDir)) {
this.error(chalk.red(`Values directory not found at ${valuesDir}`));
return;
}

let credentials: Record<string, string>;
if (provider === 'vault') {
credentials = await this.getVaultCredentials()
credentials = await this.getVaultCredentials();
} else {
credentials = await this.getAWSCredentials()
credentials = await this.getAWSCredentials();
}

for (const chart of charts) {
let yamlPath: string
if (chart === 'l2-bootnode' || chart === 'l2-sequencer') {
yamlPath = path.join(process.cwd(), chart, 'values', 'production-1.yaml')
if (!fs.existsSync(yamlPath)) {
this.log(chalk.yellow(`production-1.yaml not found for ${chart}`))
continue
}
this.log(chalk.cyan(`Using production-1.yaml for ${chart}`))
} else {
yamlPath = path.join(process.cwd(), chart, 'values', 'production.yaml')
if (!fs.existsSync(yamlPath)) {
this.log(chalk.yellow(`production.yaml not found for ${chart}`))
continue
}
this.log(chalk.cyan(`Using production.yaml for ${chart}`))
}
const yamlFiles = fs.readdirSync(valuesDir).filter(file =>
file.endsWith('-production.yaml') || file.match(/-production-\d+\.yaml$/)
);

const content = fs.readFileSync(yamlPath, 'utf8')
const yamlContent = yaml.load(content) as any
for (const yamlFile of yamlFiles) {
const yamlPath = path.join(valuesDir, yamlFile);
this.log(chalk.cyan(`Processing ${yamlFile}`));

let updated = false
const content = fs.readFileSync(yamlPath, 'utf8');
const yamlContent = yaml.load(content) as any;

let updated = false;
if (yamlContent.externalSecrets) {
for (const [secretName, secret] of Object.entries(yamlContent.externalSecrets) as [string, any][]) {
if (secret.provider !== provider) {
secret.provider = provider
updated = true
secret.provider = provider;
updated = true;
}

if (provider === 'vault') {
secret.server = credentials.server
secret.path = credentials.path
secret.version = credentials.version
secret.tokenSecretName = credentials.tokenSecretName
secret.tokenSecretKey = credentials.tokenSecretKey
delete secret.serviceAccount
delete secret.secretRegion
updated = true
secret.server = credentials.server;
secret.path = credentials.path;
secret.version = credentials.version;
secret.tokenSecretName = credentials.tokenSecretName;
secret.tokenSecretKey = credentials.tokenSecretKey;
delete secret.serviceAccount;
delete secret.secretRegion;
updated = true;
} else {
secret.serviceAccount = credentials.serviceAccount
secret.secretRegion = credentials.secretRegion
delete secret.server
delete secret.path
delete secret.version
delete secret.tokenSecretName
delete secret.tokenSecretKey
updated = true
secret.serviceAccount = credentials.serviceAccount;
secret.secretRegion = credentials.secretRegion;
delete secret.server;
delete secret.path;
delete secret.version;
delete secret.tokenSecretName;
delete secret.tokenSecretKey;
updated = true;
}

// Update remoteRef for migrate-db secrets
if (secretName.endsWith('-migrate-db')) {
for (const data of secret.data) {
if (data.remoteRef && data.remoteRef.key && data.secretKey === 'migrate-db.json') {
data.remoteRef.property = 'migrate-db.json'
updated = true
data.remoteRef.property = 'migrate-db.json';
updated = true;
}
}
}

// Update remoteRef for l2-sequencer secrets
if (secretName.match(/^l2-sequencer-\d+-secret$/)) {
for (const data of secret.data) {
if (data.remoteRef && data.remoteRef.key) {
data.remoteRef.key = 'l2-sequencer-secret';
updated = true;
}
}
}
}
}

if (updated) {
const newContent = yaml.dump(yamlContent, { lineWidth: -1, noRefs: true, quotingType: '"', forceQuotes: true })
fs.writeFileSync(yamlPath, newContent)
this.log(chalk.green(`Updated externalSecrets provider in ${chalk.cyan(chart)}/values/${path.basename(yamlPath)}`))
const newContent = yaml.dump(yamlContent, { lineWidth: -1, noRefs: true, quotingType: '"', forceQuotes: true });
fs.writeFileSync(yamlPath, newContent);
this.log(chalk.green(`Updated externalSecrets provider in ${chalk.cyan(yamlFile)}`));
} else {
this.log(chalk.yellow(`No changes needed in ${chalk.cyan(chart)}/values/${path.basename(yamlPath)}`))
this.log(chalk.yellow(`No changes needed in ${chalk.cyan(yamlFile)}`));
}
}
}


public async run(): Promise<void> {
const { flags } = await this.parse(SetupPushSecrets)
this.flags = flags

this.log(chalk.blue('Starting secret push process...'))

Expand Down
17 changes: 12 additions & 5 deletions src/commands/setup/tls.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ export default class SetupTls extends Command {
static override examples = [
'<%= config.bin %> <%= command.id %>',
'<%= config.bin %> <%= command.id %> --debug',
'<%= config.bin %> <%= command.id %> --values-dir custom-values',
]

static override flags = {
Expand All @@ -23,10 +24,15 @@ export default class SetupTls extends Command {
description: 'Show debug output and confirm before making changes',
default: false,
}),
'values-dir': Flags.string({
description: 'Directory containing the values files',
default: 'values',
}),
}

private selectedIssuer: string | null = null
private debugMode: boolean = false
private valuesDir: string = 'values'

private async checkClusterIssuer(): Promise<boolean> {
try {
Expand Down Expand Up @@ -97,11 +103,10 @@ spec:
}

private async updateChartIngress(chart: string, issuer: string): Promise<void> {
const chartPath = path.join(process.cwd(), chart)
const yamlPath = path.join(chartPath, 'values', 'production.yaml')
const yamlPath = path.join(process.cwd(), this.valuesDir, `${chart}-production.yaml`)

if (!fs.existsSync(yamlPath)) {
this.log(chalk.yellow(`production.yaml not found for ${chart}`))
this.log(chalk.yellow(`${chart}-production.yaml not found in ${this.valuesDir} directory`))
return
}

Expand Down Expand Up @@ -200,6 +205,7 @@ spec:
public async run(): Promise<void> {
const { flags } = await this.parse(SetupTls)
this.debugMode = flags.debug
this.valuesDir = flags['values-dir']

try {
this.log(chalk.blue('Starting TLS configuration update...'))
Expand Down Expand Up @@ -238,12 +244,13 @@ spec:
this.log(chalk.green(`Using ClusterIssuer: ${this.selectedIssuer}`))

const chartsToUpdate = [
'admin-system-dashboard',
'frontends',
'blockscout',
'coordinator-api',
'bridge-history-api',
'rollup-explorer-backend',
'l2-rpc'
'l2-rpc',
]

for (const chart of chartsToUpdate) {
Expand All @@ -255,4 +262,4 @@ spec:
this.error(chalk.red(`Failed to update TLS configuration: ${error}`))
}
}
}
}

0 comments on commit 2020121

Please sign in to comment.