From 12a3c53d1c04669b6bc4ce78b92ad338c755be74 Mon Sep 17 00:00:00 2001 From: Pierre Rineau Date: Thu, 14 Nov 2024 16:25:51 +0100 Subject: [PATCH] no issue - much better configuration --- CHANGELOG.md | 11 + UPGRADE.md | 25 ++ .../db_tools.standalone.complete.sample.yaml | 124 ++++++++ config/db_tools.standalone.sample.yaml | 178 ++++------- config/packages/db_tools.yaml | 162 +++++----- .../Config/AnonymizationConfig.php | 1 + src/Backupper/AbstractBackupper.php | 21 +- src/Backupper/BackupperFactory.php | 59 +--- src/Backupper/MariadbBackupper.php | 6 + src/Backupper/MysqlBackupper.php | 6 + src/Backupper/PgsqlBackupper.php | 6 + src/Backupper/SqliteBackupper.php | 6 + src/Bridge/Standalone/Bootstrap.php | 144 +++++++-- .../Standalone/StandaloneConfiguration.php | 57 ---- .../StandaloneDatabaseSessionRegistry.php | 1 + .../Compiler/DbToolsPass.php | 24 +- .../DbToolsConfiguration.php | 280 ++++++++++++------ .../DependencyInjection/DbToolsExtension.php | 64 ++-- .../Symfony/Resources/config/services.yaml | 36 ++- .../Anonymization/AnonymizeCommand.php | 6 - .../Anonymization/ConfigDumpCommand.php | 2 +- src/Command/BackupCommand.php | 2 - src/Command/RestoreCommand.php | 2 - src/Configuration/Configuration.php | 85 ++++++ src/Configuration/ConfigurationRegistry.php | 43 +++ src/Restorer/AbstractRestorer.php | 20 +- src/Restorer/MariadbRestorer.php | 6 + src/Restorer/MysqlRestorer.php | 7 + src/Restorer/PgsqlRestorer.php | 6 + src/Restorer/RestorerFactory.php | 20 +- src/Restorer/SqliteRestorer.php | 6 + src/Storage/Storage.php | 24 +- .../BackupperRestorerTest.php | 65 ++-- .../Functional/Command/RestoreCommandTest.php | 1 - .../config/packages/db_tools_alt1.yaml | 29 -- .../config/packages/db_tools_alt2.yaml | 37 --- .../db_tools_connections_partial.yaml | 22 ++ .../packages/db_tools_deprecated_v2.yaml | 15 + .../db_tools_deprecated_v2_conflict.yaml | 11 + .../config/packages/db_tools_empty.yaml | 4 + .../config/packages/db_tools_full.yaml | 40 +++ .../config/packages/db_tools_min.yaml | 1 - .../config/packages/db_tools_removed_v2.yaml | 14 + .../DbToolsConfigurationTest.php | 244 ++++++++------- .../DbToolsExtensionTest.php | 204 ++++++++++++- tests/Unit/Storage/StorageTest.php | 13 +- 46 files changed, 1407 insertions(+), 733 deletions(-) create mode 100644 UPGRADE.md create mode 100644 config/db_tools.standalone.complete.sample.yaml delete mode 100644 src/Bridge/Standalone/StandaloneConfiguration.php create mode 100644 src/Configuration/Configuration.php create mode 100644 src/Configuration/ConfigurationRegistry.php delete mode 100644 tests/Resources/config/packages/db_tools_alt1.yaml delete mode 100644 tests/Resources/config/packages/db_tools_alt2.yaml create mode 100644 tests/Resources/config/packages/db_tools_connections_partial.yaml create mode 100644 tests/Resources/config/packages/db_tools_deprecated_v2.yaml create mode 100644 tests/Resources/config/packages/db_tools_deprecated_v2_conflict.yaml create mode 100644 tests/Resources/config/packages/db_tools_empty.yaml create mode 100644 tests/Resources/config/packages/db_tools_full.yaml delete mode 100644 tests/Resources/config/packages/db_tools_min.yaml create mode 100644 tests/Resources/config/packages/db_tools_removed_v2.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index ce8b2fe1..8b21098a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,18 @@ ## Next * [feature] 🌟 Add `bin/db-tools` CLI command allowing standalone usage (#153). +* [feature] ⭐️ The CLI tool can run without configuration using only environment variables (#191). +* [feature] ⭐️ All global options can now be configured on a per-connection basis in `connections.NAME.OPTION` (#191). +* [feature] 🌟 In both CLI and Symfony, the `anonymyzation` configuration option may now directly hold the complete anonymization configuration without requiring an additional file. (#191). * [experimental] ⭐️ Add `bin/compile` CLI command for building a PHAR file (#154). +* [deprecation] `anonymization.yaml` is replaced by `anonymization_files` (#191). +* [deprecation] `excluded_tables` is replaced by either `backup_excluded_tables` or `connections.NAME.backup_excluded_tables` (#191). +* [deprecation] `storage.filename_strategy` is replaced by either `storage_filename_strategy` or `connections.NAME.filename_strategy` (#191). +* [deprecation] `storage.root_dir` is replaced by either `storage_directory` or `connections.NAME.storage_directory` (#191). +* [bc] `backupper_binaries` (array) is replaced by either `backup_binary` (string) or `connections.NAME.backup_binary` (string) (#191). +* [bc] `backupper_options` (array) is replaced by either `backup_options` (string) or `connections.NAME.backup_options` (string) (#191). +* [bc] `restorer_binaries` (array) is replaced by either `restore_binary` (string) or `connections.NAME.restore_binary` (string) (#191). +* [bc] `restorer_options` (array) is replaced by either `restore_options` (string) or `connections.NAME.restore_options` (string) (#191). * [bc] Password anonymizer `symfony/password-hasher` dependency is now optional and must be manually installed (#155). * [fix] Property must not be accessed before initialization error when using `--list` option (#183, @iNem0o). * [internal] All Doctrine related dependencies are now optional (#155). diff --git a/UPGRADE.md b/UPGRADE.md new file mode 100644 index 00000000..7483741c --- /dev/null +++ b/UPGRADE.md @@ -0,0 +1,25 @@ +# Upgrade guide + +## Next + +### Configuration file structure change + +The 2.0 version introduces many configuration changes. Most of them will +gracefuly fallback to older version until 3.0, but some have been removed +and will cause exceptions. + +Rationale is that now, all top-level configuration options can be directly +set at the connection level, and we renamed those options to be more consistent +and more explicit about what they do. + +Please read carefully the new sample configuration files: + - For Symfony: [config/packages/db_tools.yaml](./config/packages/db_tools.yaml) + - For standalone: [config/db_tools.standalone.yaml](./config/db_tools.standalone.yaml) + +And the the [CHANGELOG.md](./CHANGELOG.md) file and fix your configuration accordingly. + +The `backupper_binaries` and `backupper_options` as well as the `restorer_binaries` +and `restorer_options` options have been removed and will raise exception when +kept: older version allowed to configure backup and restore binaries on a per-vendor +basis, they are now configured on a per-connection basis without considering the +database vendor anymore. Please set those options for each connection instead. diff --git a/config/db_tools.standalone.complete.sample.yaml b/config/db_tools.standalone.complete.sample.yaml new file mode 100644 index 00000000..058ce3cb --- /dev/null +++ b/config/db_tools.standalone.complete.sample.yaml @@ -0,0 +1,124 @@ +# This configuration file is an example for standalone usage. +# Only required parameter is the connection URL. + +# Working directory is the path to which all relative file references will +# be relative to. If none set, the path will be this file directory instead. +# In most cases, you will not need it. +#workdir: . + +# Where to put generated backups. +# Root directory of the backup storage manager. Default filename +# strategy will always use this folder as root path. +#storage_directory: ./var/db_tools + +# Default filename strategy. You may specify one strategy for each doctrine +# connection. +# If you created and registered a custom one into the container as a +# service, you may simply set the service identifier. If no service +# exists, and your implementation does not require parameters, simply +# set the class name. +# Allowed values are: +# - "default": alias of "datetime". +# - "datetime": implementation is "%db_tools.storage_directory%/YYYY/MM/-.". +# - CLASS_NAME: a class name to use that implements a strategy. +#storage_filename_strategy: default + +# When old backups are considered obsolete. +# (Use relative date/time formats : https://www.php.net/manual/en/datetime.formats.relative.php) +#backup_expiration_age: '6 months ago' # default '3 months ago' + +# Default timeout for backup process. +#backup_timeout: 1200 # default 600 + +# Default timeout for restore process. +#restore_timeout: 2400 # default 1800 + +# List here tables (you don't want in your backups. +# If you have more than one connection, it is strongly advised to configure +# this for each connection instead. +#backup_excluded_tables: ['table1', 'table2'] + +# Specify here paths to backup and restore binaries and command line options. +# Warning: this will apply to all connections disregarding their database +# vendor. If you have more than one connection and if they use different +# database vendors or versions, please configure those for each connection +# instead. +# Default values depends upon vendor and are documented at +# https://dbtoolsbundle.readthedocs.io/en/stable/configuration.html +#backup_binary: '/usr/bin/pg_dump' +#backup_options: '-Z 5 --lock-wait-timeout=120' +#restore_binary: '/usr/bin/pg_restore' +#restore_options: '-j 2 --clean --if-exists --disable-triggers' + +# If you have a single connection, you can use this syntax. +# Connection name will be "default" when configured this way. +connections: "pgsql://username:password@hostname:port?version=16.0&other_option=..." + +# Alternatively, you are allowed to have more than one connections, case +# in which you might synthesize the configuration as this: +#connections: +# connection_one: "pgsql://username:password@hostname:port?version=16.0&other_option=..." +# connection_two: "mysql://username:password@hostname:port?version=8.1&other_option=..." + +# For advanced usage, you may also override any parameter for each connection. +# Each key is a connection name, all parameters above are allowed for each +# unique connection. +#connections: +# connection_one: +# # Connection URL for connecting. +# # Please refer to makinacorpus/db-query-builder or documentation for more information. +# # Any URL built for doctrine/dbal usage should work. +# # URL is the sole mandatory parameter. +# # Complete list of accepted parameters follows. +# url: "pgsql://username:password@hostname:port?version=16.0&other_option=..." +# backup_binary: /usr/local/bin/vendor-one-dump +# backup_excluded_tables: ['table_one', 'table_two'] +# backup_expiration_age: '1 month ago' +# backup_options: --no-table-lock +# backup_timeout: 2000 +# restore_binary: /usr/local/bin/vendor-one-restore +# restore_options: --disable-triggers --other-option +# restore_timeout: 5000 +# storage_directory: /path/to/storage +# storage_filename_strategy: datetime +# connection_two: +# url: "mysql://username:password@hostname:port?version=8.1&other_option=..." +# # ... + +# You can explicitely set which will be default connection in use when +# none providen in the command line options. If you omit this configuration +# value, then the first one in the list will be used. +#default_connection: connection_one + +# Update this configuration if you want to look for anonymizers in a custom folder. +#anonymizer_paths: + #- ./src/Anonymization/Anonymizer' + +# For simple needs, you may simply write the anonymization configuration here. +# Keys are connection names, values are structures which are identical to what +# you may find in the "anonymizations.sample.yaml" example. +anonymization: + connection_one: + table1: + column1: + anonymizer: anonymizer_name + # ... anonymizer specific options... + column2: + # ... + table2: + # ... + connection_two: + # ... + +# You can for organisation purpose delegate anonymization config into extra +# YAML configuration files, and simply reference them here. +# Pathes can be either relative or absolute. Relative paths are relative to +# the workdir option if specified, or from this configuration file directory +# otherwise. +# See the "anonymizations.sample.yaml" in this folder for an example. +anonymization_files: + connection_one: './anonymization/connection_one.yaml' + connection_two: './anonymization/connection_two.yaml' + +# If you have only one connection, you can adopt the following syntax. +#anonymization_files: '%kernel.project_dir%/config/anonymizations.yaml' diff --git a/config/db_tools.standalone.sample.yaml b/config/db_tools.standalone.sample.yaml index 8967853f..86e283d7 100644 --- a/config/db_tools.standalone.sample.yaml +++ b/config/db_tools.standalone.sample.yaml @@ -1,137 +1,67 @@ # This configuration file is an example for standalone usage. -# -# The given parameters you will find in this file must not be set in a Symfony -# application context, and will be ignored if so. -# -# All other configuration you can find in the ./packages/db_tools.yaml file -# can be added in this file, you must simply omit the 'db_tools:' top level -# node. +# Only required parameter is the connection URL. -# Working directory is the path to which all relative file references will -# be relative to. If none set, the path will be this file directory instead. -workdir: /var/www/my_project/ +# If you have a single connection, you can use this syntax. +# Connection name will be "default" when configured this way. +# If you have multiple connections, please refer to the exhaustive +# sample in 'db_tools.standalone.complete.sample.yaml'. +connections: "pgsql://username:password@hostname:port?version=16.0&other_option=..." -# Database connections. -# One line per connection, a single database URL, all options as query -# parameters. Connections will be made using makincorpus/query-builder -# which will raise exceptions when invalid options are found. -# There is less configuration amplitude than using doctrine/dbal in -# Symfony, yet it should be enough in most case. -# In case any options or specific behaviour is missing, please file -# an issue at https://github.com/makinacorpus/php-query-builder/issues -connections: - connection_one: "pgsql://username:password@hostname:port?version=16.0&other_option=..." - connection_two: "mysql://username:password@hostname:port?version=8.1&other_option=..." +# Where to put generated backups. +# Root directory of the backup storage manager. Default filename +# strategy will always use this folder as root path. +#storage_directory: ./var/db_tools -# If you have a single connection, you can use this syntax. In this case -# the connection name will be "default". -# connections: "pgsql://username:password@hostname:port?version=16.0&other_option=..." +# Default filename strategy. You may specify one strategy for each connection. +# If you created and registered a custom one into the container as a +# service, you may simply set the service identifier. If no service +# exists, and your implementation does not require parameters, simply +# set the class name. +# Allowed values are: +# - "default": alias of "datetime". +# - "datetime": implementation is "%db_tools.storage_directory%/YYYY/MM/-.". +# - CLASS_NAME: a class name to use that implements a strategy. +#storage_filename_strategy: default -# You can explicitely set which will be default connection in use when -# none providen in the command line options. If you omit this configuration -# value, then the first one in the list will be used. -#default_connection: connection_one - -# Using the DbToolsBundle standalone, you must provide at least -# a root directory for backups. -storage: - # Path can be relative or absolute, Relative paths are relative to the - # workdir option if specified, or from this configuration file directory - # otherwise. - # If none provided, the default will be the following one. - root_dir: ./var/db_tools - # Filename strategies. You may specify one strategy for each doctrine - # connection. Keys are doctrine connection names. Values are strategy - # names, "default" (or null) or omitting the connection will use the - # default implementation. - # If you created and registered a custom one into the container as a - # service, you may simply set the service identifier. If no service - # exists, and your implementation does not require parameters, simply - # set the class name. - #filename_strategy: - # Backup filename strategy. - # "default" is an alias of "datetime" - #default: default - # "datetime" implementation is ROOT_DIR/YYYY/MM/-." - #other_connection_strategy: datetime - # Example of using a service name: - #yet_another_connection: app.db_tools.filename.custom_strategy - # Or a classe name: - #another_one: App\DbTools\Storage\MyCustomStrategy - -# When old backups are considered obsolete +# When old backups are considered obsolete. # (Use relative date/time formats : https://www.php.net/manual/en/datetime.formats.relative.php) #backup_expiration_age: '6 months ago' # default '3 months ago' -# Timeout for backups. -# backup_timeout: 1200 # default 600 - -# Timeout for restores. -# restore_timeout: 2400 # default 1800 - -# List here tables (per connection) you don't want in your backups -#excluded_tables: - #default: ['table1', 'table2'] - -# Specify here paths to binaries, only if the system can't find them by himself -# platform are 'mysql', 'postgresql', 'sqlite' -#backupper_binaries: - #mariadb: '/usr/bin/mariadb-dump' # default 'mariadb-dump' - #mysql: '/usr/bin/mysqldump' # default 'mysqldump' - #postgresql: '/usr/bin/pg_dump' # default 'pg_dump' - #sqlite: '/usr/bin/sqlite3' # default 'sqlite3' -#restorer_binaries: - #mariadb: '/usr/bin/mariadb' # default 'mariadb' - #mysql: '/usr/bin/mysql' # default 'mysql' - #postgresql: '/usr/bin/pg_restore' # default 'pg_restore' - #sqlite: '/usr/bin/sqlite3' # default 'sqlite3' - -# Default options to pass to the binary when backing up or restoring -# a database. Those options must be defined per connection. -# If you do not define some default options, here or by using the -# "--extra-options" option when invoking the command, the following -# ones will be used according to the database vendor: -# - When backing up: -# - MariaDB: --no-tablespaces -# - MySQL: --no-tablespaces -# - PostgreSQL: -Z 5 --lock-wait-timeout=120 -# - SQLite: -bail -# - When restoring: -# - MariaDB: None -# - MySQL: None -# - PostgreSQL: -j 2 --clean --if-exists --disable-triggers -# - SQLite: None -#backupper_options: - #default: '' - #another_connection: '' -#restorer_options: - #default: '' - #another_connection: '' +# List here tables (you don't want in your backups. +# If you have more than one connection, it is strongly advised to configure +# this for each connection instead. +#backup_excluded_tables: ['table1', 'table2'] # Update this configuration if you want to look for anonymizers in a custom folder. -# These are default paths that will always be registered even if you override -# the setting and don't repeat them: #anonymizer_paths: #- ./src/Anonymization/Anonymizer' -anonymization: - # From here you can proceed with manual file inclusion. Pathes can be - # either relative or absolute. Relative paths are relative to the workdir - # option if specified, or from this configuration file directory - # otherwise. - yaml: - connection_one: ./db_tools.anonymizer.connection_one.yaml - # ... other connections ... - - # Extra configuration options, if you don't want to split the anonymization - # configuration into multiple files, you can directly write it here. - tables: - connection_one: - # From here, please refer to 'anonymizations.sample.yaml' for sample - # and documentation. - table_name: - column_name: - anonymizer: anonymizer_name - # ... other options... - connection_two: - # ... +# Write the anonymization configuration here. +# You may also write anonymization configuration in extra files instead, please +# see the 'db_tools.standalone.complete.sample.yaml' for more documentation. +#anonymization: +# # Keys are connection names, values are structures which are identical to what +# # you may find in the "anonymizations.sample.yaml" example. +# # If you only specified one URL in the "connections" parameter above, the +# # connection name is "default". +# default: +# # Keys here are table names, followed by column names, column values +# # are either an anonymizer name string, or an object with options. +# user: +# # Some Anonymizer does not require any option, you can use them like this +# prenom: fr-fr.firstname +# nom: fr-fr.lastname +# # Some do require options, specify them like this: +# age: +# anonymizer: integer +# options: {min: 0, max: 99} +# # Some others have optionnal options, you can specify those: +# email: +# anonymizer: email +# options: {domain: 'toto.com'} +# # Or leave it with defaults: +# email: email +# level: +# anonymizer: string +# options: {sample: ['none', 'bad', 'good', 'expert']} +# # ... other tables... diff --git a/config/packages/db_tools.yaml b/config/packages/db_tools.yaml index b3fd5d4d..573742f5 100644 --- a/config/packages/db_tools.yaml +++ b/config/packages/db_tools.yaml @@ -1,96 +1,104 @@ db_tools: # Where to put generated backups. - #storage: - # Root directory of the backup storage manager. Default filename - # strategy will always use this folder as root path. - #root_dir: '%kernel.project_dir%/var/db_tools' + # Root directory of the backup storage manager. Default filename + # strategy will always use this folder as root path. + #storage_directory: '%kernel.project_dir%/var/db_tools' - # Filename strategies. You may specify one strategy for each doctrine - # connection. Keys are doctrine connection names. Values are strategy - # names, "default" (or null) or omitting the connection will use the - # default implementation. - # If you created and registered a custom one into the container as a - # service, you may simply set the service identifier. If no service - # exists, and your implementation does not require parameters, simply - # set the class name. - #filename_strategy: - # Backup filename strategy. - # "default" is an alias of "datetime" - #default: default - # "datetime" implementation is "%db_tools.storage.root_dir%/YYYY/MM/-." - #other_connection_strategy: datetime - # Example of using a service name: - #yet_another_connection: app.db_tools.filename.custom_strategy - # Or a classe name: - #another_one: App\DbTools\Storage\MyCustomStrategy + # Filename strategies. You may specify one strategy for each doctrine + # connection. Keys are doctrine connection names. Values are strategy + # names, "default" (or null) or omitting the connection will use the + # default implementation. + # If you created and registered a custom one into the container as a + # service, you may simply set the service identifier. If no service + # exists, and your implementation does not require parameters, simply + # set the class name. + # Allowed values are: + # - "default": alias of "datetime". + # - "datetime": implementation is "%db_tools.storage_directory%/YYYY/MM/-.". + # - CLASS_NAME: a class name to use that implements a strategy. + # - SERVICE_ID: A service identifier registered in container that implements a strategy. + #storage_filename_strategy: default - # When old backups are considered obsolete + # When old backups are considered obsolete. # (Use relative date/time formats : https://www.php.net/manual/en/datetime.formats.relative.php) #backup_expiration_age: '6 months ago' # default '3 months ago' - # Timeout for backups. - # backup_timeout: 1200 # default 600 + # Default timeout for backup process. + #backup_timeout: 1200 # default 600 - # Timeout for restores. - # restore_timeout: 2400 # default 1800 + # Default timeout for restore process. + #restore_timeout: 2400 # default 1800 - # List here tables (per connection) you don't want in your backups - #excluded_tables: - #default: ['table1', 'table2'] + # List here tables (you don't want in your backups. + # If you have more than one connection, it is strongly advised to configure + # this for each connection instead. + #backup_excluded_tables: ['table1', 'table2'] - # Specify here paths to binaries, only if the system can't find them by himself - # platform are 'mysql', 'postgresql', 'sqlite' - #backupper_binaries: - #mariadb: '/usr/bin/mariadb-dump' # default 'mariadb-dump' - #mysql: '/usr/bin/mysqldump' # default 'mysqldump' - #postgresql: '/usr/bin/pg_dump' # default 'pg_dump' - #sqlite: '/usr/bin/sqlite3' # default 'sqlite3' - #restorer_binaries: - #mariadb: '/usr/bin/mariadb' # default 'mariadb' - #mysql: '/usr/bin/mysql' # default 'mysql' - #postgresql: '/usr/bin/pg_restore' # default 'pg_restore' - #sqlite: '/usr/bin/sqlite3' # default 'sqlite3' + # Specify here paths to backup and restore binaries and command line options. + # Warning: this will apply to all connections disregarding their database + # vendor. If you have more than one connection and if they use different + # database vendors or versions, please configure those for each connection + # instead. + # Default values depends upon vendor and are documented at + # https://dbtoolsbundle.readthedocs.io/en/stable/configuration.html + #backup_binary: '/usr/bin/pg_dump' + #backup_options: '-Z 5 --lock-wait-timeout=120' + #restore_binary: '/usr/bin/pg_restore' + #restore_options: '-j 2 --clean --if-exists --disable-triggers' - # Default options to pass to the binary when backing up or restoring - # a database. Those options must be defined per connection. - # If you do not define some default options, here or by using the - # "--extra-options" option when invoking the command, the following - # ones will be used according to the database vendor: - # - When backing up: - # - MariaDB: --no-tablespaces - # - MySQL: --no-tablespaces - # - PostgreSQL: -Z 5 --lock-wait-timeout=120 - # - SQLite: -bail - # - When restoring: - # - MariaDB: None - # - MySQL: None - # - PostgreSQL: -j 2 --clean --if-exists --disable-triggers - # - SQLite: None - #backupper_options: - #default: '' - #another_connection: '' - #restorer_options: - #default: '' - #another_connection: '' + # For advanced usage, you may also override any parameter for each connection. + # Each key is a connection name, all parameters above are allowed for each + # unique connection. + # Keys are doctrine connection names. + #connections: + # connection_one: + # # Complete list of accepted parameters follows. + # url: "pgsql://username:password@hostname:port?version=16.0&other_option=..." + # backup_binary: /usr/local/bin/vendor-one-dump + # backup_excluded_tables: ['table_one', 'table_two'] + # backup_expiration_age: '1 month ago' + # backup_options: --no-table-lock + # backup_timeout: 2000 + # restore_binary: /usr/local/bin/vendor-one-restore + # restore_options: --disable-triggers --other-option + # restore_timeout: 5000 + # storage_directory: /path/to/storage + # storage_filename_strategy: datetime + # connection_two: + # # ... # Update this configuration if you want to look for anonymizers in a custom folder. # These are default paths that will always be registered even if you override - # the setting and don't repeat them: + # the setting so please don't repeat them: #anonymizer_paths: #- '%kernel.project_dir%/vendor/makinacorpus/db-tools-bundle/src/Anonymization/Anonymizer/Core' #- '%kernel.project_dir%/src/Anonymization/Anonymizer' - # Anonymization configuration. + # For simple needs, you may simply write the anonymization configuration here. + # Keys are connection names, values are structures which are identical to what + # you may find in the "anonymizations.sample.yaml" example. #anonymization: - # If you want to configure anonymization with attributes on - # Doctrine entities, you have nothing to add here: if doctrine/orm - # is available the DbToolsBundle will automatically look for it. - # - # If you want to load configuration from a yaml: - # 1/ If you want to configure anonymization only for the default - # DBAL connection, declare it like this: - #yaml: '%kernel.project_dir%/config/anonymizations.yaml' - # 2/ If you use multiple connections, declare each configuration like this: - #yaml: - #- connection_one: '%kernel.project_dir%/config/anonymizations/connection_one.yaml' - #- connection_two: '%kernel.project_dir%/config/anonymizations/connection_two.yaml' + # connection_one: + # table1: + # column1: + # anonymizer: anonymizer_name + # # ... anonymizer specific options... + # column2: + # # ... + # table2: + # # ... + # connection_two: + # # ... + + # You can for organisation purpose delegate anonymization config into extra + # YAML configuration files, and simply reference them here. + # Pathes can be either relative or absolute. Relative paths are relative to + # the workdir option if specified, or from this configuration file directory + # otherwise. + # See the "anonymizations.sample.yaml" in this folder for an example. + #anonymization_files: + # connection_one: '%kernel.project_dir%/config/anonymization/connection_one.yaml' + # connection_two: '%kernel.project_dir%/config/anonymization/connection_two.yaml' + + # If you have only one connection, you can adopt the following syntax. + #anonymization_files: '%kernel.project_dir%/config/anonymizations.yaml' diff --git a/src/Anonymization/Config/AnonymizationConfig.php b/src/Anonymization/Config/AnonymizationConfig.php index f8bb0e46..9d4c7ca8 100644 --- a/src/Anonymization/Config/AnonymizationConfig.php +++ b/src/Anonymization/Config/AnonymizationConfig.php @@ -60,6 +60,7 @@ public function getTableConfig(string $table, ?array $filteredTargets = null): a return $config; } + /** * Count tables. */ diff --git a/src/Backupper/AbstractBackupper.php b/src/Backupper/AbstractBackupper.php index 9ff9f787..25141ca1 100644 --- a/src/Backupper/AbstractBackupper.php +++ b/src/Backupper/AbstractBackupper.php @@ -4,6 +4,7 @@ namespace MakinaCorpus\DbToolsBundle\Backupper; +use MakinaCorpus\DbToolsBundle\Configuration\Configuration; use MakinaCorpus\DbToolsBundle\Helper\Output\NullOutput; use MakinaCorpus\DbToolsBundle\Helper\Process\CommandLine; use MakinaCorpus\DbToolsBundle\Helper\Process\ProcessTrait; @@ -22,21 +23,24 @@ abstract class AbstractBackupper implements LoggerAwareInterface { use ProcessTrait; + protected string $binary; protected ?string $destination = null; protected string $defaultOptions = ''; protected ?string $extraOptions = null; protected bool $ignoreDefaultOptions = false; protected array $excludedTables = []; protected bool $verbose = false; - protected ?int $timeout = 600; + protected ?int $timeout = null; public function __construct( - protected string $binary, - protected DatabaseSession $databaseSession, - protected Dsn $databaseDsn, - ?string $defaultOptions = null, + protected readonly DatabaseSession $databaseSession, + protected readonly Dsn $databaseDsn, + protected readonly Configuration $config, ) { - $this->defaultOptions = $defaultOptions ?? $this->getBuiltinDefaultOptions(); + $this->excludedTables = $config->getBackupExcludedTables(); + $this->binary = $config->getBackupBinary() ?? $this->getDefaultBinary(); + $this->defaultOptions = $config->getBackupOptions() ?? $this->getBuiltinDefaultOptions(); + $this->timeout = $config->getBackupTimeout(); $this->destination = \sprintf( '%s/db-tools-backup-%s.dump', @@ -161,6 +165,11 @@ protected function beforeProcess(): void $this->process->setTimeout(null === $this->timeout ? null : (float) $this->timeout); } + /** + * Get default binary path and name (e.g. "/usr/bin/foosql-backup"). + */ + abstract protected function getDefaultBinary(): string; + /** * Provide the built-in default options that will be used if none is given * through the dedicated constructor argument. diff --git a/src/Backupper/BackupperFactory.php b/src/Backupper/BackupperFactory.php index b39f65fc..fe9e8dd2 100644 --- a/src/Backupper/BackupperFactory.php +++ b/src/Backupper/BackupperFactory.php @@ -4,6 +4,7 @@ namespace MakinaCorpus\DbToolsBundle\Backupper; +use MakinaCorpus\DbToolsBundle\Configuration\ConfigurationRegistry; use MakinaCorpus\DbToolsBundle\Database\DatabaseSessionRegistry; use MakinaCorpus\DbToolsBundle\Error\NotImplementedException; use MakinaCorpus\QueryBuilder\Vendor; @@ -11,62 +12,11 @@ class BackupperFactory { - /** - * Constructor. - * - * @param array $backupperBinaries - * @param array $backupperOptions - * @param array $excludedTables - */ public function __construct( private DatabaseSessionRegistry $registry, - private array $backupperBinaries, - private array $backupperOptions = [], - private array $excludedTables = [], + private ConfigurationRegistry $configRegistry = new ConfigurationRegistry(), private ?LoggerInterface $logger = null, - ) { - $connectionNames = $this->registry->getConnectionNames(); - - // Normalize vendor names otherwise automatic creation might fail. - foreach ($this->backupperBinaries as $vendorName => $binary) { - $this->backupperBinaries[Vendor::vendorNameNormalize($vendorName)] = $binary; - } - - foreach ($this->backupperOptions as $connectionName => $options) { - if (!\in_array($connectionName, $connectionNames)) { - throw new \DomainException(\sprintf( - "'%s' is not a valid connection name.", - $connectionName - )); - } - if (!\is_string($options)) { - throw new \InvalidArgumentException( - "Each value of the \$backupperOptions argument must be a string." - ); - } - } - - foreach ($this->excludedTables as $connectionName => $tableNames) { - if (!\in_array($connectionName, $connectionNames)) { - throw new \DomainException(\sprintf( - "'%s' is not a valid connection name.", - $connectionName - )); - } - if (!\is_array($tableNames)) { - throw new \InvalidArgumentException( - "Each value of the \$excludedTables argument must be an array of table names (strings)." - ); - } - foreach ($tableNames as $tableName) { - if (!\is_string($tableName)) { - throw new \InvalidArgumentException( - "Each table name given through the \$excludedTables argument must be a string." - ); - } - } - } - } + ) {} /** * Get a Backupper for the given connection. @@ -92,10 +42,9 @@ public function create(?string $connectionName = null): AbstractBackupper }; $backupper = new $backupper( - $this->backupperBinaries[$vendorName], $this->registry->getDatabaseSession($connectionName), $dsn, - $this->backupperOptions[$connectionName] ?? null + $this->configRegistry->getConnectionConfig($connectionName), ); \assert($backupper instanceof AbstractBackupper); diff --git a/src/Backupper/MariadbBackupper.php b/src/Backupper/MariadbBackupper.php index dfb2e7ec..6065b978 100644 --- a/src/Backupper/MariadbBackupper.php +++ b/src/Backupper/MariadbBackupper.php @@ -51,6 +51,12 @@ public function getExtension(): string return 'sql'; } + #[\Override] + protected function getDefaultBinary(): string + { + return 'mariadb-dump'; + } + #[\Override] protected function getBuiltinDefaultOptions(): string { diff --git a/src/Backupper/MysqlBackupper.php b/src/Backupper/MysqlBackupper.php index 176f9372..37f99349 100644 --- a/src/Backupper/MysqlBackupper.php +++ b/src/Backupper/MysqlBackupper.php @@ -51,6 +51,12 @@ public function getExtension(): string return 'sql'; } + #[\Override] + protected function getDefaultBinary(): string + { + return 'mysqldump'; + } + #[\Override] protected function getBuiltinDefaultOptions(): string { diff --git a/src/Backupper/PgsqlBackupper.php b/src/Backupper/PgsqlBackupper.php index caa8ebbe..9aff566e 100644 --- a/src/Backupper/PgsqlBackupper.php +++ b/src/Backupper/PgsqlBackupper.php @@ -60,6 +60,12 @@ public function getExtension(): string return 'dump'; } + #[\Override] + protected function getDefaultBinary(): string + { + return 'pg_dump'; + } + #[\Override] protected function getBuiltinDefaultOptions(): string { diff --git a/src/Backupper/SqliteBackupper.php b/src/Backupper/SqliteBackupper.php index 521a5388..92203945 100644 --- a/src/Backupper/SqliteBackupper.php +++ b/src/Backupper/SqliteBackupper.php @@ -36,6 +36,12 @@ public function getExtension(): string return 'sql'; } + #[\Override] + protected function getDefaultBinary(): string + { + return 'sqlite3'; + } + #[\Override] protected function getBuiltinDefaultOptions(): string { diff --git a/src/Bridge/Standalone/Bootstrap.php b/src/Bridge/Standalone/Bootstrap.php index 83200355..16c41ad2 100644 --- a/src/Bridge/Standalone/Bootstrap.php +++ b/src/Bridge/Standalone/Bootstrap.php @@ -7,16 +7,20 @@ use Composer\InstalledVersions; use MakinaCorpus\DbToolsBundle\Anonymization\AnonymizatorFactory; use MakinaCorpus\DbToolsBundle\Anonymization\Anonymizer\AnonymizerRegistry; +use MakinaCorpus\DbToolsBundle\Anonymization\Config\Loader\ArrayLoader; +use MakinaCorpus\DbToolsBundle\Anonymization\Config\Loader\YamlLoader; use MakinaCorpus\DbToolsBundle\Backupper\BackupperFactory; use MakinaCorpus\DbToolsBundle\Bridge\Symfony\DependencyInjection\DbToolsConfiguration; -use MakinaCorpus\DbToolsBundle\Command\Anonymization\AnonymizeCommand; -use MakinaCorpus\DbToolsBundle\Command\Anonymization\AnonymizerListCommand; -use MakinaCorpus\DbToolsBundle\Command\Anonymization\CleanCommand; -use MakinaCorpus\DbToolsBundle\Command\Anonymization\ConfigDumpCommand; use MakinaCorpus\DbToolsBundle\Command\BackupCommand; use MakinaCorpus\DbToolsBundle\Command\CheckCommand; use MakinaCorpus\DbToolsBundle\Command\RestoreCommand; use MakinaCorpus\DbToolsBundle\Command\StatsCommand; +use MakinaCorpus\DbToolsBundle\Command\Anonymization\AnonymizeCommand; +use MakinaCorpus\DbToolsBundle\Command\Anonymization\AnonymizerListCommand; +use MakinaCorpus\DbToolsBundle\Command\Anonymization\CleanCommand; +use MakinaCorpus\DbToolsBundle\Command\Anonymization\ConfigDumpCommand; +use MakinaCorpus\DbToolsBundle\Configuration\Configuration; +use MakinaCorpus\DbToolsBundle\Configuration\ConfigurationRegistry; use MakinaCorpus\DbToolsBundle\Database\DatabaseSessionRegistry; use MakinaCorpus\DbToolsBundle\Error\ConfigurationException; use MakinaCorpus\DbToolsBundle\Restorer\RestorerFactory; @@ -75,9 +79,13 @@ public static function createApplication(): Application // We need to parse a few arguments prior running the console // application in order to setup commands. This is hackish but // should work. + // @todo This does not work, fix it. $config = $configFiles = []; if ($input->hasOption('config')) { foreach ((array) $input->getOption('config') as $filename) { + if ($output->isVerbose()) { + $output->writeln('Using configuration file: ' . $filename); + } $configFiles[] = $filename; } } @@ -190,22 +198,56 @@ public static function bootstrap(array $config = [], array $configFiles = [], ?L $logger ?? new NullLogger(); $config = self::configParse($config, $configFiles, $logger); - $databaseSessionRegistry = self::createDatabaseSessionRegistry($config); + $default = new Configuration( + backupBinary: $config['backup_binary'] ?? null, + backupExcludedTables: $config['backup_excluded_tables'] ?? null, + backupExpirationAge: $config['backup_expiration_age'] ?? null, + backupOptions: $config['backup_options'] ?? null, + backupTimeout: $config['backup_timeout'] ?? null, + restoreBinary: $config['restore_binary'] ?? null, + restoreOptions: $config['restore_options'] ?? null, + restoreTimeout: $config['restore_timeout'] ?? null, + storageDirectory: $config['storage_directory'] ?? null, + storageFilenameStrategy: $config['storage_filename_strategy'] ?? null, + ); + + $connections = []; + foreach (($config['connections'] ?? []) as $name => $data) { + $connections[$name] = new Configuration( + backupBinary: $data['backup_binary'] ?? null, + backupExcludedTables: $data['backup_excluded_tables'] ?? null, + backupExpirationAge: $data['backup_expiration_age'] ?? null, + backupOptions: $data['backup_options'] ?? null, + backupTimeout: $data['backup_timeout'] ?? null, + restoreBinary: $data['restore_binary'] ?? null, + restoreOptions: $data['restore_options'] ?? null, + restoreTimeout: $data['restore_timeout'] ?? null, + parent: $default, + storageDirectory: $data['storage_directory'] ?? null, + storageFilenameStrategy: $data['storage_filename_strategy'] ?? null, + url: $data['url'] ?? null, + ); + } + + $configRegistry = new ConfigurationRegistry($default, $connections, $config['default_connection'] ?? null); + + $databaseSessionRegistry = self::createDatabaseSessionRegistry($configRegistry); $anonymizerRegistry = self::createAnonymizeRegistry($config); $anonymizatorFactory = new AnonymizatorFactory($databaseSessionRegistry, $anonymizerRegistry, $logger); - $backupperBinaries = $config['backupper_binaries']; - $backupperExcludedTables = $config['excluded_tables'] ?? []; - $backupperOptions = $config['backupper_options']; - $backupperFactory = new BackupperFactory($databaseSessionRegistry, $backupperBinaries, $backupperOptions, $backupperExcludedTables, $logger); + foreach (($config['anonymization_files'] ?? []) as $connectionName => $file) { + $anonymizatorFactory->addConfigurationLoader(new YamlLoader($file, $connectionName)); + } + foreach (($config['anonymization'] ?? []) as $connectionName => $array) { + $anonymizatorFactory->addConfigurationLoader(new ArrayLoader($array, $connectionName)); + } - $restorerBinaries = $config['restorer_binaries']; - $restorerOptions = $config['restorer_options']; - $restorerFactory = new RestorerFactory($databaseSessionRegistry, $restorerBinaries, $restorerOptions, $logger); + $backupperFactory = new BackupperFactory($databaseSessionRegistry, $configRegistry, $logger); + $restorerFactory = new RestorerFactory($databaseSessionRegistry, $configRegistry, $logger); $statsProviderFactory = new StatsProviderFactory($databaseSessionRegistry); - $storage = self::createStorage($config, $logger); + $storage = self::createStorage($configRegistry, $logger); return new Context( anonymizatorFactory: $anonymizatorFactory, @@ -312,15 +354,17 @@ private static function configParse(array $config, array $files, LoggerInterface $configs[] = self::configParseFile($filename); } $configs[] = $config; - $configs[] = self::configGetEnv(); + + $config = self::configGetEnv($config); // Use symfony/config and our bundle configuration, which allows us // to use it fully for validation and merge. - $configuration = new StandaloneConfiguration(); + $configuration = new DbToolsConfiguration(true, true); $processor = new Processor(); $config = $processor->processConfiguration($configuration, $configs); $config = DbToolsConfiguration::appendPostConfig($config); + $config = DbToolsConfiguration::fixLegacyOptions($config); return $config; } @@ -353,19 +397,19 @@ private static function configParseFile(string $filename): array $workdir = \rtrim($config['workdir'] ?? \dirname($filename), '/'); // Storage root directory. - if ($path = ($config['storage']['root_dir'] ?? null)) { - $config['storage']['root_dir'] = self::pathAbs($workdir, $path); + if ($path = ($config['storage_directory'] ?? null)) { + $config['storage_directory'] = self::pathAbs($workdir, $path); } // YAML anonymizer file paths. - $yaml = $config['anonymization']['yaml'] ?? null; + $yaml = $config['anonymization_files'] ?? null; if (isset($yaml)) { if (\is_array($yaml)) { foreach ($yaml as $name => $path) { - $config['anonymization']['yaml'][$name] = self::pathAbs($workdir, $path); + $config['anonymization_files'][$name] = self::pathAbs($workdir, $path); } } else { - $config['anonymization']['yaml'] = self::pathAbs($workdir, $yaml); + $config['anonymization_files'] = self::pathAbs($workdir, $yaml); } } @@ -380,13 +424,49 @@ private static function configParseFile(string $filename): array /** * Get config variables from environment variables. */ - private static function configGetEnv(): array + private static function configGetEnv(array $config): array { - $config = []; + if (!isset($config['backup_binary'])) { + $config['backup_binary'] = self::getEnv('DBTOOLS_BACKUP_BINARY'); + } + if (!isset($config['backup_excluded_tables'])) { + $config['backup_excluded_tables'] = self::getEnv('DBTOOLS_BACKUP_EXCLUDED_TABLES'); + } + if (!isset($config['backup_expiration_age'])) { + $config['backup_expiration_age'] = self::getEnv('DBTOOLS_BACKUP_EXPIRATION_AGE'); + } + if (!isset($config['backup_options'])) { + $config['backup_options'] = self::getEnv('DBTOOLS_BACKUP_OPTIONS'); + } + if (!isset($config['backup_timeout'])) { + $config['backup_timeout'] = self::getEnv('DBTOOLS_BACKUP_TIMEOUT'); + } + if (!isset($config['default_connection'])) { + $config['default_connection'] = self::getEnv('DBTOOLS_DEFAULT_CONNECTION'); + } + if (!isset($config['restore_binary'])) { + $config['restore_binary'] = self::getEnv('DBTOOLS_RESTORE_BINARY'); + } + if (!isset($config['restore_options'])) { + $config['restore_options'] = self::getEnv('DBTOOLS_RESTORE_OPTIONS'); + } + if (!isset($config['restore_timeout'])) { + $config['restore_timeout'] = self::getEnv('DBTOOLS_RESTORE_TIMEOUT'); + } + if (!isset($config['storage_directory'])) { + $config['storage_directory'] = self::getEnv('DBTOOLS_STORAGE_DIRECTORY'); + } + if (!isset($config['storage_filename_strategy'])) { + $config['storage_filename_strategy'] = self::getEnv('DBTOOLS_STORAGE_FILENAME_STRATEGY'); + } + return $config; + } - // @todo read env variables, validate each, override $config + private static function getEnv(string $name): string|null + { + $value = \getenv($name); - return $config; + return (!$value && $value !== '0') ? null : (string) $value; } /** @@ -400,20 +480,26 @@ private static function createAnonymizeRegistry(array $config): AnonymizerRegist /** * Create database session registry from config-given connections. */ - private static function createDatabaseSessionRegistry(array $config): DatabaseSessionRegistry + private static function createDatabaseSessionRegistry(ConfigurationRegistry $configRegistry): DatabaseSessionRegistry { + $connections = []; + foreach ($configRegistry->getConnectionConfigAll() as $name => $config) { + \assert($config instanceof Configuration); + $connections[$name] = $config->getUrl() ?? throw new ConfigurationException(\sprintf('Connection "%s" is missing the "url" option.', $name)); + } + // Do not crash on initialization, it will crash later when a connection // will be request instead: this allows commands that don't act on // database (such as anonymizer list) to work even if not configured. - return new StandaloneDatabaseSessionRegistry($config['connections'] ?? [], $config['default_connection']); + return new StandaloneDatabaseSessionRegistry($connections, $configRegistry->getDefaultConnection()); } /** * Create storage. */ - private static function createStorage(array $config, LoggerInterface $logger): Storage + private static function createStorage(ConfigurationRegistry $configRegistry, LoggerInterface $logger): Storage { - $rootdir = $config['storage']['root_dir'] ?? $config['workdir']; + $rootdir = $configRegistry->getDefaultConfig()->getStorageDirectory(); if (!\is_dir($rootdir)) { if (\file_exists($rootdir)) { @@ -425,7 +511,7 @@ private static function createStorage(array $config, LoggerInterface $logger): S $logger->notice("Found storage root folder: {dir}", ['dir' => $rootdir]); } - return new Storage($config['storage']['root_dir'], $config['backup_expiration_age']); + return new Storage($configRegistry); } /** diff --git a/src/Bridge/Standalone/StandaloneConfiguration.php b/src/Bridge/Standalone/StandaloneConfiguration.php deleted file mode 100644 index b71cef09..00000000 --- a/src/Bridge/Standalone/StandaloneConfiguration.php +++ /dev/null @@ -1,57 +0,0 @@ -getRootNode() - ->children() - ->scalarNode('workdir') - ->info('Directory path all other files will be relative to, if none providen then the configuration file directory will be used instead.') - ->defaultNull() - ->end() - ->arrayNode('connections') - ->beforeNormalization()->ifString()->then(function ($v) { return ['default' => $v]; })->end() - ->scalarPrototype() - ->info('Database connection DSN/URL.') - ->end() - ->end() - ->scalarNode('default_connection') - ->info('Default connection name. If none providen, first one is used instead.') - ->defaultNull() - ->end() - ->arrayNode('anonymization') - ->children() - ->arrayNode('tables') - ->beforeNormalization()->ifString()->then(function ($v) { return ['default' => $v]; })->end() - ->variablePrototype() - ->info('Keys are table names, values are arrays whose keys are column names and values are anonymizer configurations.') - ->end() - ->end() - ->end() - ->end() - ->end() - ->end() - ; - - return $treeBuilder; - } -} diff --git a/src/Bridge/Standalone/StandaloneDatabaseSessionRegistry.php b/src/Bridge/Standalone/StandaloneDatabaseSessionRegistry.php index 0d319ed0..e06c64be 100644 --- a/src/Bridge/Standalone/StandaloneDatabaseSessionRegistry.php +++ b/src/Bridge/Standalone/StandaloneDatabaseSessionRegistry.php @@ -37,6 +37,7 @@ public function getDefaultConnectionName(): string foreach (\array_keys($this->connections) as $name) { return $this->defaultConnection = $name; } + throw new ConfigurationException("No connections were configured."); } return $this->defaultConnection; } diff --git a/src/Bridge/Symfony/DependencyInjection/Compiler/DbToolsPass.php b/src/Bridge/Symfony/DependencyInjection/Compiler/DbToolsPass.php index 48fc8758..b4a2982f 100644 --- a/src/Bridge/Symfony/DependencyInjection/Compiler/DbToolsPass.php +++ b/src/Bridge/Symfony/DependencyInjection/Compiler/DbToolsPass.php @@ -4,6 +4,7 @@ namespace MakinaCorpus\DbToolsBundle\Bridge\Symfony\DependencyInjection\Compiler; +use MakinaCorpus\DbToolsBundle\Anonymization\Config\Loader\ArrayLoader; use MakinaCorpus\DbToolsBundle\Anonymization\Config\Loader\AttributesLoader; use MakinaCorpus\DbToolsBundle\Anonymization\Config\Loader\YamlLoader; use MakinaCorpus\DbToolsBundle\Bridge\Symfony\DependencyInjection\DbToolsConfiguration; @@ -28,15 +29,34 @@ public function process(ContainerBuilder $container): void $anonymazorFactoryDef->addMethodCall('addConfigurationLoader', [new Reference($loaderId)]); } - if (isset($config['anonymization']) && isset($config['anonymization']['yaml'])) { - foreach ($config['anonymization']['yaml'] as $connectionName => $file) { + if (isset($config['anonymization_files'])) { + foreach ($config['anonymization_files'] as $connectionName => $file) { $loaderId = $this->registerYamlLoader($file, $connectionName, $container); $anonymazorFactoryDef->addMethodCall('addConfigurationLoader', [new Reference($loaderId)]); } } + + if (isset($config['anonymization'])) { + foreach ($config['anonymization'] as $connectionName => $data) { + $loaderId = $this->registerArrayLoader($data, $connectionName, $container); + $anonymazorFactoryDef->addMethodCall('addConfigurationLoader', [new Reference($loaderId)]); + } + } } } + private function registerArrayLoader(array $data, string $connectionName, ContainerBuilder $container): string + { + $definition = new Definition(); + $definition->setClass(ArrayLoader::class); + $definition->setArguments([$data, $connectionName]); + + $loaderId = 'db_tools.anonymization.loader.array.' . $connectionName; + $container->setDefinition($loaderId, $definition); + + return $loaderId; + } + private function registerYamlLoader(string $file, string $connectionName, ContainerBuilder $container): string { $definition = new Definition(); diff --git a/src/Bridge/Symfony/DependencyInjection/DbToolsConfiguration.php b/src/Bridge/Symfony/DependencyInjection/DbToolsConfiguration.php index 56f1698f..ef665963 100644 --- a/src/Bridge/Symfony/DependencyInjection/DbToolsConfiguration.php +++ b/src/Bridge/Symfony/DependencyInjection/DbToolsConfiguration.php @@ -5,17 +5,15 @@ namespace MakinaCorpus\DbToolsBundle\Bridge\Symfony\DependencyInjection; use Symfony\Component\Config\Definition\ConfigurationInterface; +use Symfony\Component\Config\Definition\Builder\ArrayNodeDefinition; use Symfony\Component\Config\Definition\Builder\TreeBuilder; class DbToolsConfiguration implements ConfigurationInterface { - /** - * Default storage path cannot use variable when standalone. - */ - protected function getDefaultStoragePath(): ?string - { - return '%kernel.project_dir%/var/db_tools'; - } + public function __construct( + protected readonly bool $standalone = false, + protected readonly bool $withDeprecated = false, + ) {} /** * Append values in configuration we cannot set a default. @@ -34,109 +32,225 @@ public static function appendPostConfig(array $config): array return $config; } - #[\Override] - public function getConfigTreeBuilder(): TreeBuilder + /** + * From a normalized configuration array, fixes legacy options names, + * and raise errors when duplicates are found. + * + * @internal + * @todo Alter in 3.x accordingly to option removal. + */ + public static function fixLegacyOptions(array $config): array { - $treeBuilder = new TreeBuilder('db_tools'); - - $intervalToInt = function (mixed $v): null|int { - if (null === $v) { - return $v; + if (isset($config['storage']['root_dir'])) { + \trigger_deprecation('makinacorpus/db-tools-bundle', '2.0.0', '"db_tools.storage.root_dir" configuration option is deprecated and renamed "db_tools.storage_directory"'); + if (isset($config['storage_directory'])) { + throw throw new \InvalidArgumentException('Deprecated option "storage.root_dir" and actual option "storage_directory" are both defined, please fix your configuration.'); } - if (\is_string($v)) { - if (\ctype_digit($v)) { - return (int) $v; - } - try { - if (false !== ($i = @\DateInterval::createFromDateString($v))) { - return $i->days * 86400 + $i->h * 3600 + $i->i * 60 + $i->s; - } - } catch (\DateMalformedIntervalStringException) { - // Pass, invalid format. + $config['storage_directory'] = $config['storage']['root_dir']; + } + if (isset($config['storage']['filename_strategy'])) { + \trigger_deprecation('makinacorpus/db-tools-bundle', '2.0.0', '"db_tools.storage.filename_strategy" configuration option is deprecated and renamed "db_tools.storage_filename_strategy"'); + foreach ($config['storage']['filename_strategy'] as $connection => $strategy) { + \trigger_deprecation('makinacorpus/db-tools-bundle', '2.0.0', '"storage.filename_strategy.%s" configuration option is deprecated and renamed "connections.%s.storage_filename_strategy"', $connection, $connection); + if (isset($config['connections'][$connection]['storage_filename_strategy'])) { + throw throw new \InvalidArgumentException(\sprintf('Deprecated option "storage.filename_strategy.%s" and actual option "connections.%s.storage_filename_strategy" are both defined, please fix your configuration.', $connection, $connection)); } - throw throw new \InvalidArgumentException(\sprintf("Given value '%s' is not an int and cannot be parsed as a date interval", $v)); + $config['connections'][$connection]['storage_filename_strategy'] = $strategy; } - if (\is_int($v) || \is_float($v)) { - return (int) $v; + } + unset($config['storage']); + + if (isset($config['excluded_tables'])) { + foreach ($config['excluded_tables'] as $connection => $tables) { + \trigger_deprecation('makinacorpus/db-tools-bundle', '2.0.0', '"excluded_tables.%s" configuration option is deprecated and renamed "connections.%s.backup_excluded_tables"', $connection, $connection); + if (isset($config['connections'][$connection]['backup_excluded_tables'])) { + throw throw new \InvalidArgumentException(\sprintf('Deprecated option "excluded_tables.%s" and actual option "connections.%s.backup_excluded_tables" are both defined, please fix your configuration.', $connection, $connection)); + } + $config['connections'][$connection]['backup_excluded_tables'] = $tables; } - throw throw new \InvalidArgumentException(\sprintf("Expected an int or valid date interval string value, got '%s'", \get_debug_type($v))); - }; + } + unset($config['excluded_tables']); + + if (isset($config['anonymization']['yaml'])) { + \trigger_deprecation('makinacorpus/db-tools-bundle', '2.0.0', '"db_tools.anonymization.yaml" configuration option is deprecated and renamed "anonymization_files"'); + $config['anonymization_files'] = $config['anonymization']['yaml']; + } + unset($config['anonymization']['yaml']); + + return $config; + } + + #[\Override] + public function getConfigTreeBuilder(): TreeBuilder + { + $treeBuilder = new TreeBuilder('db_tools'); + $rootNode = $treeBuilder->getRootNode(); + // For PHPStan. + \assert($rootNode instanceof ArrayNodeDefinition); // @phpstan-ignore-next-line - $treeBuilder - ->getRootNode() + $rootNode + ->children() + ->arrayNode('anonymization') + ->useAttributeAsKey('connection') + ->variablePrototype() + ->info('Keys are table names, values are arrays whose keys are column names and values are anonymizer configurations.') + ->end() + ->end() + ->arrayNode('anonymization_files') + ->useAttributeAsKey('connection') + ->beforeNormalization()->ifString()->then(function ($v) { return ['default' => $v]; })->end() + ->scalarPrototype()->end() + ->end() + ->arrayNode('anonymizer_paths') + ->defaultValue([]) + ->scalarPrototype()->end() + ->end() + ->end() + ; + + // Add defaults. + $this->addConnectionConfigTreeBuilder($rootNode); + + // Add "connections" children definition. + // @phpstan-ignore-next-line + $connectionsNode = $rootNode + ->children() + ->arrayNode('connections') + ->useAttributeAsKey('name') + ->arrayPrototype() + ->children() + ->scalarNode('url') + ->defaultNull() + ->end() + ->end() + // Do not close arrayNode() we use it below. + ; + \assert($connectionsNode instanceof ArrayNodeDefinition); + $this->addConnectionConfigTreeBuilder($connectionsNode); + + if ($this->standalone) { + // Add extra options for standalone CLI app. + // @phpstan-ignore-next-line + $rootNode ->children() - ->scalarNode('storage_directory') - ->setDeprecated('makinacorpus/db-tools-bundle', '1.0.1', 'Please use "db_tools.storage.root_dir" instead.') + ->scalarNode('workdir') + ->info('Directory path all other files will be relative to, if none providen then the configuration file directory will be used instead.') + ->defaultNull() ->end() + ->scalarNode('default_connection') + ->info('Default connection name. If none providen, first one is used instead.') + ->defaultNull() + ->end() + ->end() + ; + } + + if ($this->withDeprecated) { + // Add deprecated options. + // @phpstan-ignore-next-line + $rootNode + ->children() ->arrayNode('storage') ->addDefaultsIfNotSet() ->children() - ->scalarNode('root_dir')->defaultValue($this->getDefaultStoragePath())->end() + ->scalarNode('root_dir') + ->setDeprecated('makinacorpus/db-tools-bundle', '2.0.0', 'Please use "storage_root_dir" instead.') + ->defaultNull() + ->end() ->arrayNode('filename_strategy') + ->setDeprecated('makinacorpus/db-tools-bundle', '2.0.0', 'Please use "storage_filename_strategy" instead.') + ->beforeNormalization()->ifString()->then(function ($v) { return ['default' => $v]; })->end() ->useAttributeAsKey('connection') ->scalarPrototype()->end() ->end() ->end() ->end() - ->scalarNode('backup_expiration_age')->defaultValue('3 months ago')->end() - ->scalarNode('backup_timeout') - ->beforeNormalization()->always($intervalToInt)->end() - ->defaultValue(600) - ->end() - ->scalarNode('restore_timeout') - ->beforeNormalization()->always($intervalToInt)->end() - ->defaultValue(1800) - ->end() + // @todo Remove in 3.x ->arrayNode('excluded_tables') + ->setDeprecated('makinacorpus/db-tools-bundle', '2.0.0', 'Please use "db_tools.backup_excluded_tables" instead.') + ->beforeNormalization()->always(function ($v) { return \array_is_list($v) ? ['default' => $v] : $v; })->end() ->useAttributeAsKey('connection') ->arrayPrototype() ->scalarPrototype()->end() ->end() ->end() - ->arrayNode('backupper_binaries') - ->defaultValue([ - 'mariadb' => 'mariadb-dump', - 'mysql' => 'mysqldump', - 'postgresql' => 'pg_dump', - 'sqlite' => 'sqlite3', - ]) - ->scalarPrototype()->end() - ->end() - ->arrayNode('restorer_binaries') - ->defaultValue([ - 'mariadb' => 'mariadb', - 'mysql' => 'mysql', - 'postgresql' => 'pg_restore', - 'sqlite' => 'sqlite3', - ]) - ->scalarPrototype()->end() - ->end() - ->arrayNode('backupper_options') - ->useAttributeAsKey('connection') - ->scalarPrototype()->end() - ->end() - ->arrayNode('restorer_options') - ->useAttributeAsKey('connection') - ->scalarPrototype()->end() - ->end() - ->arrayNode('anonymizer_paths') - ->defaultValue([]) - ->scalarPrototype()->end() - ->end() - ->arrayNode('anonymization') - ->children() - ->arrayNode('yaml') - ->useAttributeAsKey('connection') - ->beforeNormalization()->ifString()->then(function ($v) { return ['default' => $v]; })->end() - ->scalarPrototype()->end() - ->end() - ->end() - ->end() + ->end() + ; + } + + return $treeBuilder; + } + + /** + * Common options for connections and top-level default configuration. + */ + protected function addConnectionConfigTreeBuilder(ArrayNodeDefinition $node): void + { + $intervalToInt = $this->getIntervalToInt(); + + // @phpstan-ignore-next-line + $node + ->children() + ->scalarNode('backup_binary') + ->defaultNull() + ->end() + ->arrayNode('backup_excluded_tables') + ->beforeNormalization()->always(function ($v) { return \is_array($v) ? $v : [$v]; })->end() + ->scalarPrototype()->end() + ->end() + ->scalarNode('backup_expiration_age')->end() + ->scalarNode('backup_options') + ->defaultNull() + ->end() + ->scalarNode('backup_timeout') + ->beforeNormalization()->always($intervalToInt)->end() + ->end() + ->scalarNode('restore_binary') + ->defaultNull() + ->end() + ->scalarNode('restore_options') + ->defaultNull() + ->end() + ->scalarNode('restore_timeout') + ->beforeNormalization()->always($intervalToInt)->end() + ->end() + ->scalarNode('storage_directory') + ->defaultNull() + ->end() + ->scalarNode('storage_filename_strategy') + ->defaultNull() ->end() ->end() ; + } - return $treeBuilder; + /** + * Convert any value to an interval as a number of second. + */ + protected function getIntervalToInt(): callable + { + return function (mixed $v): null|int { + if (null === $v) { + return $v; + } + if (\is_string($v)) { + if (\ctype_digit($v)) { + return (int) $v; + } + try { + if (false !== ($i = @\DateInterval::createFromDateString($v))) { + return $i->days * 86400 + $i->h * 3600 + $i->i * 60 + $i->s; + } + } catch (\DateMalformedIntervalStringException) { + // Pass, invalid format. + } + throw throw new \InvalidArgumentException(\sprintf("Given value '%s' is not an int and cannot be parsed as a date interval", $v)); + } + if (\is_int($v) || \is_float($v)) { + return (int) $v; + } + throw throw new \InvalidArgumentException(\sprintf("Expected an int or valid date interval string value, got '%s'", \get_debug_type($v))); + }; } } diff --git a/src/Bridge/Symfony/DependencyInjection/DbToolsExtension.php b/src/Bridge/Symfony/DependencyInjection/DbToolsExtension.php index bc2b1298..741c8605 100644 --- a/src/Bridge/Symfony/DependencyInjection/DbToolsExtension.php +++ b/src/Bridge/Symfony/DependencyInjection/DbToolsExtension.php @@ -4,15 +4,16 @@ namespace MakinaCorpus\DbToolsBundle\Bridge\Symfony\DependencyInjection; +use MakinaCorpus\DbToolsBundle\Configuration\Configuration; use MakinaCorpus\DbToolsBundle\Storage\FilenameStrategyInterface; -use Symfony\Component\Config\Definition\ConfigurationInterface; use Symfony\Component\Config\FileLocator; +use Symfony\Component\Config\Definition\ConfigurationInterface; use Symfony\Component\DependencyInjection\ContainerBuilder; use Symfony\Component\DependencyInjection\Definition; +use Symfony\Component\DependencyInjection\Reference; use Symfony\Component\DependencyInjection\Exception\InvalidArgumentException; use Symfony\Component\DependencyInjection\Extension\Extension; use Symfony\Component\DependencyInjection\Loader\YamlFileLoader; -use Symfony\Component\DependencyInjection\Reference; final class DbToolsExtension extends Extension { @@ -21,29 +22,52 @@ public function load(array $configs, ContainerBuilder $container): void { $configuration = $this->getConfiguration($configs, $container); $config = $this->processConfiguration($configuration, $configs); + + $config = DbToolsConfiguration::fixLegacyOptions($config); $config = DbToolsConfiguration::appendPostConfig($config); $loader = new YamlFileLoader($container, new FileLocator(\dirname(__DIR__).'/Resources/config')); $loader->load('services.yaml'); - if (isset($config['storage_directory'])) { - \trigger_deprecation('makinacorpus/db-tools-bundle', '1.0.1', '"db_tools.storage_directory" configuration option is deprecated and renamed "db_tools.storage.root_dir"'); - $container->setParameter('db_tools.storage.root_dir', $config['storage_directory']); - } else { - $container->setParameter('db_tools.storage.root_dir', $config['storage']['root_dir']); - } + $configDef = new Definition(); + $configDef->setClass(Configuration::class); + $configDef->setArguments([ + '$backupBinary' => $config['backup_binary'] ?? '%env(resolve:DBTOOLS_BACKUP_BINARY)%', + '$backupExcludedTables' => $config['backup_excluded_tables'] ?? null, // new Parameter('env(resolve:array:DBTOOLS_BACKUP_EXCLUDED_TABLES)'), // @todo + '$backupExpirationAge' => $config['backup_expiration_age'] ?? '%env(resolve:string:DBTOOLS_BACKUP_EXPIRATION_AGE)%', + '$backupOptions' => $config['backup_options'] ?? '%env(resolve:DBTOOLS_BACKUP_OPTIONS)%', + '$backupTimeout' => $config['backup_timeout'] ?? '%env(int:DBTOOLS_BACKUP_TIMEOUT)%', + '$parent' => null, // For Symfony 6.x. + '$restoreBinary' => $config['restore_binary'] ?? '%env(resolve:DBTOOLS_RESTORE_BINARY)%', + '$restoreOptions' => $config['restore_options'] ?? '%env(resolve:DBTOOLS_RESTORE_OPTIONS)%', + '$restoreTimeout' => $config['restore_timeout'] ?? '%env(int:DBTOOLS_RESTORE_TIMEOUT)%', + '$storageDirectory' => $config['storage_directory'] ?? '%env(resolve:DBTOOLS_STORAGE_DIRECTORY)%', + '$storageFilenameStrategy' => $config['storage_filename_strategy'] ?? '%env(resolve:DBTOOLS_STORAGE_FILENAME_STRATEGY)%', + ]); + $container->setDefinition('db_tools.configuration.default', $configDef); - // Backupper - $container->setParameter('db_tools.backupper.binaries', $config['backupper_binaries']); - $container->setParameter('db_tools.backupper.options', $config['backupper_options']); - $container->setParameter('db_tools.backup_expiration_age', $config['backup_expiration_age']); - $container->setParameter('db_tools.excluded_tables', $config['excluded_tables'] ?? []); - $container->setParameter('db_tools.backup_timeout', $config['backup_timeout']); + $connectionDefs = []; + foreach ($config['connections'] as $name => $data) { + $connConfigDef = new Definition(); + $connConfigDef->setClass(Configuration::class); + $connConfigDef->setArguments([ + '$backupBinary' => $data['backup_binary'] ?? null, + '$backupExcludedTables' => $data['backup_excluded_tables'] ?? null, + '$backupExpirationAge' => $data['backup_expiration_age'] ?? null, + '$backupOptions' => $data['backup_options'] ?? null, + '$backupTimeout' => $data['backup_timeout'] ?? null, + '$restoreBinary' => $data['restore_binary'] ?? null, + '$restoreOptions' => $data['restore_options'] ?? null, + '$restoreTimeout' => $data['restore_timeout'] ?? null, + '$parent' => new Reference('db_tools.configuration.default'), + '$storageDirectory' => $data['storage_directory'] ?? null, + '$storageFilenameStrategy' => $data['storage_filename_strategy'] ?? null, + ]); + $container->setDefinition('db_tools.configuration.connection.' . $name, $connConfigDef); + $connectionDefs[$name] = new Reference('db_tools.configuration.connection.' . $name); + } - // Restorer - $container->setParameter('db_tools.restorer.binaries', $config['restorer_binaries']); - $container->setParameter('db_tools.restorer.options', $config['restorer_options']); - $container->setParameter('db_tools.restore_timeout', $config['restore_timeout']); + $container->getDefinition('db_tools.configuration.registry')->setArguments([new Reference('db_tools.configuration.default'), $connectionDefs]); // Validate user-given anonymizer paths. $anonymizerPaths = $config['anonymizer_paths']; @@ -83,13 +107,13 @@ public function load(array $configs, ContainerBuilder $container): void } } if ($strategies) { - $container->getDefinition('db_tools.storage')->setArgument(2, $strategies); + $container->getDefinition('db_tools.storage')->setArgument(1, $strategies); } } #[\Override] public function getConfiguration(array $config, ContainerBuilder $container): ?ConfigurationInterface { - return new DbToolsConfiguration(); + return new DbToolsConfiguration(true, false); } } diff --git a/src/Bridge/Symfony/Resources/config/services.yaml b/src/Bridge/Symfony/Resources/config/services.yaml index e06e165b..10e6c391 100644 --- a/src/Bridge/Symfony/Resources/config/services.yaml +++ b/src/Bridge/Symfony/Resources/config/services.yaml @@ -1,3 +1,22 @@ +parameters: + # All allowed environement variables. + # When conflicting with options in the Symfony configuration, the Symfony + # configuration will override the environment variables. + env(DBTOOLS_BACKUP_BINARY): ~ + env(DBTOOLS_BACKUP_EXCLUDED_TABLES): ~ + env(DBTOOLS_BACKUP_EXPIRATION_AGE): ~ + env(DBTOOLS_BACKUP_OPTIONS): ~ + env(DBTOOLS_BACKUP_TIMEOUT): ~ + env(DBTOOLS_DEFAULT_CONNECTION): ~ + env(DBTOOLS_RESTORE_BINARY): ~ + env(DBTOOLS_RESTORE_OPTIONS): ~ + env(DBTOOLS_RESTORE_TIMEOUT): ~ + env(DBTOOLS_STORAGE_FILENAME_STRATEGY): ~ + env(DBTOOLS_STORAGE_DIRECTORY): "%kernel.project_dir%/var/db_tools" + + # Will be rewritten by the container extension. + db_tools.anonymization.anonymizer.paths: [] + services: # Commands db_tools.command.anonymization.run: @@ -8,8 +27,6 @@ services: - '@db_tools.backupper.factory' - '@db_tools.anonymization.anonymizator.factory' - '@db_tools.storage' - - '%db_tools.backup_timeout%' - - '%db_tools.restore_timeout%' tags: ['console.command'] db_tools.command.anonymization.list: class: MakinaCorpus\DbToolsBundle\Command\Anonymization\AnonymizerListCommand @@ -33,7 +50,6 @@ services: - '%doctrine.default_connection%' - '@db_tools.backupper.factory' - '@db_tools.storage' - - '%db_tools.backup_timeout%' tags: ['console.command'] db_tools.command.check: class: MakinaCorpus\DbToolsBundle\Command\CheckCommand @@ -48,7 +64,6 @@ services: - '%doctrine.default_connection%' - '@db_tools.restorer.factory' - '@db_tools.storage' - - '%db_tools.restore_timeout%' tags: ['console.command'] db_tools.command.stats: class: MakinaCorpus\DbToolsBundle\Command\StatsCommand @@ -57,6 +72,10 @@ services: - '@db_tools.stats_provider.factory' tags: ['console.command'] + # Configuration, arguments are populated using the extension. + db_tools.configuration.registry: + class: MakinaCorpus\DbToolsBundle\Configuration\ConfigurationRegistry + # Database registry. db_tools.database_session.registry: class: MakinaCorpus\DbToolsBundle\Bridge\Symfony\DoctrineDatabaseSessionRegistry @@ -65,16 +84,14 @@ services: # Utilities db_tools.storage: class: MakinaCorpus\DbToolsBundle\Storage\Storage - arguments: ['%db_tools.storage.root_dir%', '%db_tools.backup_expiration_age%'] + arguments: ['@db_tools.configuration.registry', []] # Backuppers db_tools.backupper.factory: class: MakinaCorpus\DbToolsBundle\Backupper\BackupperFactory arguments: - '@db_tools.database_session.registry' - - '%db_tools.backupper.binaries%' - - '%db_tools.backupper.options%' - - '%db_tools.excluded_tables%' + - '@db_tools.configuration.registry' - '@logger' tags: [{ name: 'monolog.logger', channel: 'db_tools_backup' }] @@ -83,8 +100,7 @@ services: class: MakinaCorpus\DbToolsBundle\Restorer\RestorerFactory arguments: - '@db_tools.database_session.registry' - - '%db_tools.restorer.binaries%' - - '%db_tools.restorer.options%' + - '@db_tools.configuration.registry' - '@logger' tags: [{ name: monolog.logger, channel: db_tools_restoration }] diff --git a/src/Command/Anonymization/AnonymizeCommand.php b/src/Command/Anonymization/AnonymizeCommand.php index c29a05e5..91adf735 100644 --- a/src/Command/Anonymization/AnonymizeCommand.php +++ b/src/Command/Anonymization/AnonymizeCommand.php @@ -46,8 +46,6 @@ public function __construct( private BackupperFactory $backupperFactory, private AnonymizatorFactory $anonymizatorFactory, private Storage $storage, - private ?int $backupTimeout = null, - private ?int $restoreTimeout = null, ) { parent::__construct(); } @@ -243,7 +241,6 @@ private function doBackupInitialDatabase(): void ->setDestination($this->initialBackupFilename) ->setOutput(new ConsoleOutput($this->io)) ->setVerbose($this->io->isVerbose()) - ->setTimeout($this->backupTimeout) ->execute() ; @@ -269,7 +266,6 @@ private function doRestoreGivenBackup(): void ->setBackupFilename($this->backupFilename) ->setOutput(new ConsoleOutput($this->io)) ->setVerbose($this->io->isVerbose()) - ->setTimeout($this->restoreTimeout) ->execute() ; @@ -323,7 +319,6 @@ private function doBackupAnonymizedDatabase(): void ->setDestination($this->backupFilename) ->setOutput(new ConsoleOutput($this->io)) ->setVerbose($this->io->isVerbose()) - ->setTimeout($this->backupTimeout) ->execute() ; @@ -349,7 +344,6 @@ private function doRestoreInitialDatabase(): void ->setBackupFilename($this->initialBackupFilename) ->setOutput(new ConsoleOutput($this->io)) ->setVerbose($this->io->isVerbose()) - ->setTimeout($this->restoreTimeout) ->execute() ; diff --git a/src/Command/Anonymization/ConfigDumpCommand.php b/src/Command/Anonymization/ConfigDumpCommand.php index 028d9da2..f3bbe141 100644 --- a/src/Command/Anonymization/ConfigDumpCommand.php +++ b/src/Command/Anonymization/ConfigDumpCommand.php @@ -54,7 +54,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $config->options->toDisplayString() . ( \key_exists($config->targetName, $tableErrors) ? - \PHP_EOL . '' . $tableErrors[$config->targetName] . '' + '' . $tableErrors[$config->targetName] . '' : '' ), ], diff --git a/src/Command/BackupCommand.php b/src/Command/BackupCommand.php index c2697d16..af4f87a9 100644 --- a/src/Command/BackupCommand.php +++ b/src/Command/BackupCommand.php @@ -30,7 +30,6 @@ public function __construct( private string $connectionName, private BackupperFactory $backupperFactory, private Storage $storage, - private ?int $timeout = null, ) { parent::__construct(); } @@ -121,7 +120,6 @@ private function doBackup(): string ->ignoreDefaultOptions($this->ignoreDefaultOptions) ->setOutput(new ConsoleOutput($this->io)) ->setVerbose($this->io->isVerbose()) - ->setTimeout($this->timeout) ->execute() ; diff --git a/src/Command/RestoreCommand.php b/src/Command/RestoreCommand.php index cb37dbaa..19ab7c25 100644 --- a/src/Command/RestoreCommand.php +++ b/src/Command/RestoreCommand.php @@ -30,7 +30,6 @@ public function __construct( private string $connectionName, private RestorerFactory $restorerFactory, private Storage $storage, - private ?int $timeout = null, ) { parent::__construct(); } @@ -163,7 +162,6 @@ private function doRestore(): void ->ignoreDefaultOptions($this->ignoreDefaultOptions) ->setOutput(new ConsoleOutput($this->io)) ->setVerbose($this->io->isVerbose()) - ->setTimeout($this->timeout) ->execute() ; diff --git a/src/Configuration/Configuration.php b/src/Configuration/Configuration.php new file mode 100644 index 00000000..2b26e7a3 --- /dev/null +++ b/src/Configuration/Configuration.php @@ -0,0 +1,85 @@ + */ + private readonly ?array $backupExcludedTables = [], + private readonly ?string $backupExpirationAge = null, + private readonly ?string $backupOptions = null, + private readonly null|int|string $backupTimeout = null, + private readonly ?string $restoreBinary = null, + private readonly ?string $restoreOptions = null, + private readonly null|int|string $restoreTimeout = null, + private readonly ?string $storageDirectory = null, + private readonly ?string $storageFilenameStrategy = null, + private readonly ?string $url = null, + ) {} + + public function getBackupBinary(): ?string + { + return $this->backupBinary ?? $this->parent?->getBackupBinary(); + } + + public function getBackupExcludedTables(): array + { + return $this->backupExcludedTables ?? $this->parent?->getBackupExcludedTables() ?? []; + } + + public function getBackupExpirationAge(): string + { + return $this->backupExpirationAge ?? $this->parent?->getBackupExpirationAge() ?? self::DEFAULT_BACKUP_EXPIRATION_AGE; + } + + public function getBackupOptions(): ?string + { + return $this->backupOptions ?? $this->parent?->getBackupOptions(); + } + + public function getBackupTimeout(): int + { + return $this->backupTimeout ?? $this->parent?->getBackupTimeout() ?? self::DEFAULT_BACKUP_TIMEOUT; + } + + public function getRestoreBinary(): ?string + { + return $this->restoreBinary ?? $this->parent?->getRestoreBinary(); + } + + public function getRestoreOptions(): ?string + { + return $this->restoreOptions ?? $this->parent?->getRestoreOptions(); + } + + public function getRestoreTimeout(): int + { + return $this->restoreTimeout ?? $this->parent?->getRestoreTimeout() ?? self::DEFAULT_RESTORE_TIMEOUT; + } + + public function getStorageDirectory(): string + { + return $this->storageDirectory ?? $this->parent?->getStorageDirectory() ?? './var/db_tools'; + } + + public function getStorageFilenameStrategy(): string + { + return $this->storageFilenameStrategy ?? $this->parent?->getStorageFilenameStrategy() ?? self::DEFAULT_STORAGE_FILENAME_STRATEGY; + } + + public function getUrl(): ?string + { + return $this->url; + } +} diff --git a/src/Configuration/ConfigurationRegistry.php b/src/Configuration/ConfigurationRegistry.php new file mode 100644 index 00000000..3ed54efd --- /dev/null +++ b/src/Configuration/ConfigurationRegistry.php @@ -0,0 +1,43 @@ + */ + private array $connections = [], + private ?string $defaultConnection = null, + ) {} + + public function getDefaultConfig(): Configuration + { + return $this->default ??= new Configuration(); + } + + public function getDefaultConnection(): ?string + { + if ($this->defaultConnection) { + return $this->defaultConnection; + } + + foreach (\array_keys($this->connections) as $name) { + return $name; + } + + return null; + } + + public function getConnectionConfigAll(): array + { + return $this->connections; + } + + public function getConnectionConfig(string $name): Configuration + { + return $this->connections[$name] ?? $this->getDefaultConfig(); + } +} diff --git a/src/Restorer/AbstractRestorer.php b/src/Restorer/AbstractRestorer.php index 80ac35c9..e8a253b8 100644 --- a/src/Restorer/AbstractRestorer.php +++ b/src/Restorer/AbstractRestorer.php @@ -4,6 +4,7 @@ namespace MakinaCorpus\DbToolsBundle\Restorer; +use MakinaCorpus\DbToolsBundle\Configuration\Configuration; use MakinaCorpus\DbToolsBundle\Helper\Output\NullOutput; use MakinaCorpus\DbToolsBundle\Helper\Process\CommandLine; use MakinaCorpus\DbToolsBundle\Helper\Process\ProcessTrait; @@ -20,22 +21,24 @@ abstract class AbstractRestorer implements LoggerAwareInterface { use ProcessTrait; + protected string $binary; protected ?string $backupFilename = null; protected string $defaultOptions = ''; protected ?string $extraOptions = null; protected bool $ignoreDefaultOptions = false; protected bool $verbose = false; - protected ?int $timeout = 1800; + protected ?int $timeout = null; public function __construct( - protected string $binary, - protected DatabaseSession $databaseSession, - protected Dsn $databaseDsn, - ?string $defaultOptions = null, + protected readonly DatabaseSession $databaseSession, + protected readonly Dsn $databaseDsn, + protected readonly Configuration $config, ) { - $this->defaultOptions = $defaultOptions ?? $this->getBuiltinDefaultOptions(); + $this->binary = $config->getRestoreBinary() ?? $this->getDefaultBinary(); + $this->defaultOptions = $config->getRestoreBinary() ?? $this->getBuiltinDefaultOptions(); $this->logger = new NullLogger(); $this->output = new NullOutput(); + $this->timeout = $config->getRestoreTimeout(); } /** @@ -127,6 +130,11 @@ protected function beforeProcess(): void $this->process->setTimeout(null === $this->timeout ? null : (float) $this->timeout); } + /** + * Get default binary path and name (e.g. "/usr/bin/foosql-restore"). + */ + abstract protected function getDefaultBinary(): string; + /** * Provide the built-in default options that will be used if none is given * through the dedicated constructor argument. diff --git a/src/Restorer/MariadbRestorer.php b/src/Restorer/MariadbRestorer.php index b40e1638..e8e7a731 100644 --- a/src/Restorer/MariadbRestorer.php +++ b/src/Restorer/MariadbRestorer.php @@ -60,6 +60,12 @@ protected function afterProcess(): void \fclose($this->backupStream); } + #[\Override] + protected function getDefaultBinary(): string + { + return 'mariadb'; + } + #[\Override] public function getExtension(): string { diff --git a/src/Restorer/MysqlRestorer.php b/src/Restorer/MysqlRestorer.php index b5319d98..82a44719 100644 --- a/src/Restorer/MysqlRestorer.php +++ b/src/Restorer/MysqlRestorer.php @@ -60,6 +60,13 @@ protected function afterProcess(): void \fclose($this->backupStream); } + + #[\Override] + protected function getDefaultBinary(): string + { + return 'mysql'; + } + #[\Override] public function getExtension(): string { diff --git a/src/Restorer/PgsqlRestorer.php b/src/Restorer/PgsqlRestorer.php index 75c0b99a..eecb9fb3 100644 --- a/src/Restorer/PgsqlRestorer.php +++ b/src/Restorer/PgsqlRestorer.php @@ -50,6 +50,12 @@ public function getExtension(): string return 'dump'; } + #[\Override] + protected function getDefaultBinary(): string + { + return 'pg_restore'; + } + #[\Override] protected function getBuiltinDefaultOptions(): string { diff --git a/src/Restorer/RestorerFactory.php b/src/Restorer/RestorerFactory.php index a1a3d81a..e887bb69 100644 --- a/src/Restorer/RestorerFactory.php +++ b/src/Restorer/RestorerFactory.php @@ -4,6 +4,7 @@ namespace MakinaCorpus\DbToolsBundle\Restorer; +use MakinaCorpus\DbToolsBundle\Configuration\ConfigurationRegistry; use MakinaCorpus\DbToolsBundle\Database\DatabaseSessionRegistry; use MakinaCorpus\DbToolsBundle\Error\NotImplementedException; use MakinaCorpus\QueryBuilder\Vendor; @@ -11,23 +12,11 @@ class RestorerFactory { - /** - * Constructor. - * - * @param array $restorerBinaries - * @param array $restorerOptions - */ public function __construct( private DatabaseSessionRegistry $registry, - private array $restorerBinaries, - private array $restorerOptions = [], + private ConfigurationRegistry $configRegistry = new ConfigurationRegistry(), private ?LoggerInterface $logger = null, - ) { - // Normalize vendor names otherwise automatic creation might fail. - foreach ($this->restorerBinaries as $vendorName => $binary) { - $this->restorerBinaries[Vendor::vendorNameNormalize($vendorName)] = $binary; - } - } + ) {} /** * Get a Restorer for given connection @@ -53,10 +42,9 @@ public function create(?string $connectionName = null): AbstractRestorer }; $restorer = new $restorer( - $this->restorerBinaries[$vendorName], $this->registry->getDatabaseSession($connectionName), $dsn, - $this->restorerOptions[$connectionName] ?? null + $this->configRegistry->getConnectionConfig($connectionName), ); \assert($restorer instanceof AbstractRestorer); diff --git a/src/Restorer/SqliteRestorer.php b/src/Restorer/SqliteRestorer.php index 69b3530f..80293ead 100644 --- a/src/Restorer/SqliteRestorer.php +++ b/src/Restorer/SqliteRestorer.php @@ -40,6 +40,12 @@ protected function afterProcess(): void $this->databaseSession->close(); } + #[\Override] + protected function getDefaultBinary(): string + { + return 'sqlite3'; + } + #[\Override] public function getExtension(): string { diff --git a/src/Storage/Storage.php b/src/Storage/Storage.php index 2b8c57d8..9d693d5a 100644 --- a/src/Storage/Storage.php +++ b/src/Storage/Storage.php @@ -4,6 +4,7 @@ namespace MakinaCorpus\DbToolsBundle\Storage; +use MakinaCorpus\DbToolsBundle\Configuration\ConfigurationRegistry; use Symfony\Component\Filesystem\Filesystem; use Symfony\Component\Finder\Finder; @@ -14,12 +15,9 @@ class Storage { public function __construct( - private string $storagePath, - private string $expirationAge, + private ConfigurationRegistry $configReg, private ?array $filenameStrategies = null, - ) { - $this->storagePath = \rtrim($storagePath, '/'); - } + ) {} public function listBackups( string $connectionName = 'default', @@ -27,11 +25,14 @@ public function listBackups( bool $onlyExpired = false, string $preserveFile = null ): array { + $config = $this->configReg->getConnectionConfig($connectionName); + $storagePath = \rtrim($config->getStorageDirectory(), '/'); + // In order to avoid listing dumps from other connections, we must // filter files using the connection name infix. When a custom strategy // is provided, there is no way to reproduce this filtering, it's up to // the user to give a restricted folder name. - $rootDir = $this->getFilenameStrategy($connectionName)->getRootDir($this->storagePath, $connectionName); + $rootDir = $this->getFilenameStrategy($connectionName)->getRootDir($storagePath, $connectionName); if (!(new Filesystem())->exists($rootDir)) { return []; @@ -45,7 +46,7 @@ public function listBackups( ->sortByName() ; - $expirationDate = new \Datetime($this->expirationAge); + $expirationDate = new \Datetime($config->getBackupExpirationAge()); $list = []; foreach ($finder as $file) { @@ -64,8 +65,11 @@ public function listBackups( public function generateFilename(string $connectionName = 'default', string $extension = 'sql', bool $anonymized = false): string { + $config = $this->configReg->getConnectionConfig($connectionName); + $storagePath = \rtrim($config->getStorageDirectory(), '/'); + $strategy = $this->getFilenameStrategy($connectionName); - $rootDir = $strategy->getRootDir($this->storagePath, $connectionName); + $rootDir = $strategy->getRootDir($storagePath, $connectionName); $filename = \rtrim($rootDir, '/') . '/' . $strategy->generateFilename($connectionName, $extension, $anonymized); @@ -76,11 +80,11 @@ public function generateFilename(string $connectionName = 'default', string $ext public function getStoragePath(): string { - return $this->storagePath; + return $this->configReg->getDefaultConfig()->getStorageDirectory(); } protected function getFilenameStrategy(string $connectionName): FilenameStrategyInterface { - return $this->filenameStrategies[$connectionName] ?? new DefaultFilenameStrategy(); + return $this->filenameStrategies[$connectionName] ?? $this->filenameStrategies['default'] ?? new DefaultFilenameStrategy(); } } diff --git a/tests/Functional/BackupperRestorer/BackupperRestorerTest.php b/tests/Functional/BackupperRestorer/BackupperRestorerTest.php index 3cefc22b..05206699 100644 --- a/tests/Functional/BackupperRestorer/BackupperRestorerTest.php +++ b/tests/Functional/BackupperRestorer/BackupperRestorerTest.php @@ -5,6 +5,8 @@ namespace MakinaCorpus\DbToolsBundle\Tests\Functional\BackupperRestorer; use MakinaCorpus\DbToolsBundle\Backupper\BackupperFactory; +use MakinaCorpus\DbToolsBundle\Configuration\Configuration; +use MakinaCorpus\DbToolsBundle\Configuration\ConfigurationRegistry; use MakinaCorpus\DbToolsBundle\Error\NotImplementedException; use MakinaCorpus\DbToolsBundle\Restorer\RestorerFactory; use MakinaCorpus\DbToolsBundle\Test\FunctionalTestCase; @@ -69,40 +71,14 @@ protected function createTestData(): void ); } - /** - * Get backup binary file names. - */ - private function getBinaryConfigBackup(): array - { - return [ - 'mariadb' => 'mariadb-dump', - 'mysql' => 'mysqldump', - 'postgresql' => 'pg_dump', - 'sqlite' => 'sqlite3', - ]; - } - - /** - * Get restore binary file names. - */ - private function getBinaryConfigRestore(): array - { - return [ - 'mariadb' => 'mariadb', - 'mysql' => 'mysql', - 'postgresql' => 'pg_restore', - 'sqlite' => 'sqlite3', - ]; - } - private function getBackupperFactory(): BackupperFactory { - return new BackupperFactory($this->getDatabaseSessionRegistry(), $this->getBinaryConfigBackup()); + return new BackupperFactory($this->getDatabaseSessionRegistry(), new ConfigurationRegistry()); } private function getRestorerFactory(): RestorerFactory { - return new RestorerFactory($this->getDatabaseSessionRegistry(), $this->getBinaryConfigRestore()); + return new RestorerFactory($this->getDatabaseSessionRegistry(), new ConfigurationRegistry()); } public function testBackupper(): void @@ -270,7 +246,7 @@ public function testCreateBackupper(): void { $this->skipIfDatabase(Vendor::SQLSERVER); - $backupperFactory = new BackupperFactory($this->getDatabaseSessionRegistry(), $this->getBinaryConfigBackup()); + $backupperFactory = new BackupperFactory($this->getDatabaseSessionRegistry(), new ConfigurationRegistry()); $backupper = $backupperFactory->create(); @@ -289,11 +265,16 @@ public function testCreateAnonymizerWithDefaultOptions(): void $backupperFactory = new BackupperFactory( $this->getDatabaseSessionRegistry(), - $this->getBinaryConfigBackup(), - [ - 'default' => '--fake-opt --mock-opt', - 'another' => '-x -y -z', - ] + new ConfigurationRegistry( + connections: [ + 'default' => new Configuration( + backupOptions: '--fake-opt --mock-opt', + ), + 'another' => new Configuration( + backupOptions: '-x -y -z', + ), + ] + ), ); $backupper = $backupperFactory->create(); @@ -320,12 +301,16 @@ public function testCreateAnonymizerWithExcludedTables(): void $backupperFactory = new BackupperFactory( $this->getDatabaseSessionRegistry(), - $this->getBinaryConfigBackup(), - [], - [ - 'default' => ['table1', 'table2'], - 'another' => ['table3', 'table4'], - ] + new ConfigurationRegistry( + connections: [ + 'default' => new Configuration( + backupExcludedTables: ['table1', 'table2'], + ), + 'another' => new Configuration( + backupExcludedTables: ['table3', 'table4'], + ), + ] + ), ); $backupper = $backupperFactory->create(); diff --git a/tests/Functional/Command/RestoreCommandTest.php b/tests/Functional/Command/RestoreCommandTest.php index 101fa551..93f3de07 100644 --- a/tests/Functional/Command/RestoreCommandTest.php +++ b/tests/Functional/Command/RestoreCommandTest.php @@ -65,7 +65,6 @@ private function createCommandTester(array $backupFiles = []): CommandTester 'default', $restorerFactory, $storage, - null // timeout ); return new CommandTester($command); diff --git a/tests/Resources/config/packages/db_tools_alt1.yaml b/tests/Resources/config/packages/db_tools_alt1.yaml deleted file mode 100644 index 4c8632c1..00000000 --- a/tests/Resources/config/packages/db_tools_alt1.yaml +++ /dev/null @@ -1,29 +0,0 @@ -db_tools: - storage_directory: '%kernel.project_dir%/var/backup' - backup_expiration_age: '6 months ago' - backup_timeout: 1200 - restore_timeout: 2400 - excluded_tables: - default: ['table1', 'table2'] - - backupper_binaries: - mariadb: '/usr/bin/mariadb-dump' - mysql: '/usr/bin/mysqldump' - postgresql: '/usr/bin/pg_dump' - sqlite: '/usr/bin/sqlite3' - restorer_binaries: - mariadb: '/usr/bin/mariadb' - mysql: '/usr/bin/mysql' - postgresql: '/usr/bin/pg_restore' - sqlite: '/usr/bin/sqlite3' - - backupper_options: - default: '--opt1 val1 -x -y -z --opt2 val2' - restorer_options: - default: '-abc -x val1 -y val2' - - anonymizer_paths: - - '%kernel.project_dir%/src/Anonymization/Anonymizer' - - anonymization: - yaml: '%kernel.project_dir%/config/anonymization.yaml' diff --git a/tests/Resources/config/packages/db_tools_alt2.yaml b/tests/Resources/config/packages/db_tools_alt2.yaml deleted file mode 100644 index eeecd731..00000000 --- a/tests/Resources/config/packages/db_tools_alt2.yaml +++ /dev/null @@ -1,37 +0,0 @@ -db_tools: - storage: - root_dir: '%kernel.project_dir%/var/backup' - filename_strategy: - connection_two: app.db_tools.custom_filename_strategy - backup_expiration_age: '6 months ago' - backup_timeout: 1800 - restore_timeout: 3200 - excluded_tables: - connection_two: ['table1', 'table2'] - - backupper_binaries: - mariadb: '/usr/bin/mariadb-dump' - mysql: '/usr/bin/mysqldump' - postgresql: '/usr/bin/pg_dump' - sqlite: '/usr/bin/sqlite3' - restorer_binaries: - mariadb: '/usr/bin/mariadb' - mysql: '/usr/bin/mysql' - postgresql: '/usr/bin/pg_restore' - sqlite: '/usr/bin/sqlite3' - - backupper_options: - connection_one: '--opt1 val1 -x -y -z --opt2 val2' - # Let's say we have no options for connection_two. - #connection_two: '' - restorer_options: - connection_one: '-abc -x val1 -y val2' - connection_two: '-a "Value 1" -bc -d val2 --end' - - anonymizer_paths: - - '%kernel.project_dir%/src/Anonymization/Anonymizer' - - anonymization: - yaml: - connection_one: '%kernel.project_dir%/config/anonymizations/connection_one.yaml' - connection_two: '%kernel.project_dir%/config/anonymizations/connection_two.yaml' diff --git a/tests/Resources/config/packages/db_tools_connections_partial.yaml b/tests/Resources/config/packages/db_tools_connections_partial.yaml new file mode 100644 index 00000000..465085dc --- /dev/null +++ b/tests/Resources/config/packages/db_tools_connections_partial.yaml @@ -0,0 +1,22 @@ +# +# Tests that, when using a partial connection, that unset values resolve from the defaults. +# +db_tools: + backup_binary: '/path/to/dump' + backup_excluded_tables: ['table1', 'table2'] + backup_expiration_age: '2 minutes ago' + backup_options: '--dump' + backup_timeout: 135 + connections: + connection_one: + storage_directory: '/one/storage' + storage_filename_strategy: one_strategy + backup_expiration_age: '1 minutes ago' + restore_timeout: 23 + backup_excluded_tables: ['one1'] + default_connection: connection_one + restore_binary: '/path/to/restore' + restore_options: '--restore' + restore_timeout: 357 + storage_directory: '%kernel.project_dir%/var/db_tools' + storage_filename_strategy: datetime diff --git a/tests/Resources/config/packages/db_tools_deprecated_v2.yaml b/tests/Resources/config/packages/db_tools_deprecated_v2.yaml new file mode 100644 index 00000000..3597bb27 --- /dev/null +++ b/tests/Resources/config/packages/db_tools_deprecated_v2.yaml @@ -0,0 +1,15 @@ +# +# Deprecated but yet supported configuration options in ^2.0. +# +db_tools: + anonymization: + yaml: + connection_one: 'connection_one.yaml' + connection_two: 'connection_two.yaml' + excluded_tables: + connection_one: ['one1', 'one2'] + connection_two: ['two1', 'two2', 'two3'] + storage: + filename_strategy: + connection_one: 'some_strategy' + root_dir: '/foo/bar' diff --git a/tests/Resources/config/packages/db_tools_deprecated_v2_conflict.yaml b/tests/Resources/config/packages/db_tools_deprecated_v2_conflict.yaml new file mode 100644 index 00000000..48dddeea --- /dev/null +++ b/tests/Resources/config/packages/db_tools_deprecated_v2_conflict.yaml @@ -0,0 +1,11 @@ +# +# Deprecated but yet supported configuration options in ^2.0. +# +# Conflicts between old and new name will raise exceptions. +# +db_tools: + connections: + connection_one: + backup_excluded_tables: ['one3'] + excluded_tables: + connection_one: ['one1', 'one2'] diff --git a/tests/Resources/config/packages/db_tools_empty.yaml b/tests/Resources/config/packages/db_tools_empty.yaml new file mode 100644 index 00000000..a12fa635 --- /dev/null +++ b/tests/Resources/config/packages/db_tools_empty.yaml @@ -0,0 +1,4 @@ +# +# Empty configuration. +# +db_tools: \ No newline at end of file diff --git a/tests/Resources/config/packages/db_tools_full.yaml b/tests/Resources/config/packages/db_tools_full.yaml new file mode 100644 index 00000000..ba4ea466 --- /dev/null +++ b/tests/Resources/config/packages/db_tools_full.yaml @@ -0,0 +1,40 @@ +# +# Full configuration. +# +db_tools: + anonymization: + connection_one: + user: + last_name: fr-fr.firstname + email: + anonymizer: email + options: {domain: 'toto.com'} + anonymization_files: + connection_one: 'connection_one.yaml' + connection_two: 'connection_two.yaml' + anonymizer_paths: + - './' + backup_binary: '/path/to/dump' + backup_excluded_tables: ['table1', 'table2'] + backup_expiration_age: '2 minutes ago' + backup_options: '--dump' + backup_timeout: 135 + connections: + connection_one: + storage_directory: '/one/storage' + storage_filename_strategy: one_strategy + backup_expiration_age: '1 minutes ago' + backup_timeout: 11 + restore_timeout: 23 + backup_excluded_tables: ['one1'] + backup_binary: '/path/to/dump/one' + backup_options: '--dump-one' + restore_binary: '/paht/to/restore/one' + restore_options: '--restore-one' + default_connection: connection_one + restore_binary: '/path/to/restore' + restore_options: '--restore' + restore_timeout: 357 + storage_directory: '%kernel.project_dir%/var/db_tools' + storage_filename_strategy: datetime + #workdir: /path/to diff --git a/tests/Resources/config/packages/db_tools_min.yaml b/tests/Resources/config/packages/db_tools_min.yaml deleted file mode 100644 index 0b27cc57..00000000 --- a/tests/Resources/config/packages/db_tools_min.yaml +++ /dev/null @@ -1 +0,0 @@ -db_tools: diff --git a/tests/Resources/config/packages/db_tools_removed_v2.yaml b/tests/Resources/config/packages/db_tools_removed_v2.yaml new file mode 100644 index 00000000..f1f21a13 --- /dev/null +++ b/tests/Resources/config/packages/db_tools_removed_v2.yaml @@ -0,0 +1,14 @@ +# +# Removed configuration options in ^2.0. +# +# Note: only the first will raise an error. +# +db_tools: + backupper_binaries: + postgresql: /usr/bin/pg_dump + backupper_options: + postgresql: '--foo' + restorer_binaries: + postgresql: /usr/bin/pg_restore + restorer_options: + postgresql: '--foo' diff --git a/tests/Unit/Bridge/Symfony/DependencyInjection/DbToolsConfigurationTest.php b/tests/Unit/Bridge/Symfony/DependencyInjection/DbToolsConfigurationTest.php index 7b6b0800..0b9e2f5a 100644 --- a/tests/Unit/Bridge/Symfony/DependencyInjection/DbToolsConfigurationTest.php +++ b/tests/Unit/Bridge/Symfony/DependencyInjection/DbToolsConfigurationTest.php @@ -14,157 +14,195 @@ class DbToolsConfigurationTest extends TestCase private function processYamlConfiguration(array|string $dataOrFilename): array { $processor = new Processor(); - $configuration = new DbToolsConfiguration(); + $configuration = new DbToolsConfiguration(true, true); - return $processor->processConfiguration( + $config = $processor->processConfiguration( $configuration, \is_string($dataOrFilename) ? Yaml::parseFile($dataOrFilename) : ['db_tools' => $dataOrFilename], ); + + return DbToolsConfiguration::fixLegacyOptions($config); } - public function testConfigurationMinimal(): array + public function testConfigurationEmpty(): array { $result = $this->processYamlConfiguration( - \dirname(__DIR__, 4) . '/Resources/config/packages/db_tools_min.yaml' + \dirname(__DIR__, 4) . '/Resources/config/packages/db_tools_empty.yaml' ); - self::assertSame( + self::assertEquals( [ - 'storage' => [ - 'root_dir' => '%kernel.project_dir%/var/db_tools', - 'filename_strategy' => [], - ], - 'backup_expiration_age' => '3 months ago', - 'backup_timeout' => 600, - 'restore_timeout' => 1800, - 'excluded_tables' => [], - 'backupper_binaries' => [ - 'mariadb' => 'mariadb-dump', - 'mysql' => 'mysqldump', - 'postgresql' => 'pg_dump', - 'sqlite' => 'sqlite3', - ], - 'restorer_binaries' => [ - 'mariadb' => 'mariadb', - 'mysql' => 'mysql', - 'postgresql' => 'pg_restore', - 'sqlite' => 'sqlite3', - ], - 'backupper_options' => [], - 'restorer_options' => [], + 'anonymization' => [], + 'anonymization_files' => [], 'anonymizer_paths' => [], + 'backup_binary' => null, + 'backup_excluded_tables' => [], + 'backup_options' => null, + 'connections' => [], + 'default_connection' => null, + 'restore_binary' => null, + 'restore_options' => null, + 'storage_directory' => null, + 'storage_filename_strategy' => null, + 'workdir' => null, ], - $result + $result, ); return $result; } - public function testConfigurationAlternative1(): array + public function testConfigurationFull(): array { $result = $this->processYamlConfiguration( - \dirname(__DIR__, 4) . '/Resources/config/packages/db_tools_alt1.yaml' + \dirname(__DIR__, 4) . '/Resources/config/packages/db_tools_full.yaml' ); - self::assertSame( + self::assertEquals( [ - 'storage_directory' => '%kernel.project_dir%/var/backup', - 'backup_expiration_age' => '6 months ago', - 'backup_timeout' => 1200, - 'restore_timeout' => 2400, - 'excluded_tables' => [ - 'default' => ['table1', 'table2'], - ], - 'backupper_binaries' => [ - 'mariadb' => '/usr/bin/mariadb-dump', - 'mysql' => '/usr/bin/mysqldump', - 'postgresql' => '/usr/bin/pg_dump', - 'sqlite' => '/usr/bin/sqlite3', - ], - 'restorer_binaries' => [ - 'mariadb' => '/usr/bin/mariadb', - 'mysql' => '/usr/bin/mysql', - 'postgresql' => '/usr/bin/pg_restore', - 'sqlite' => '/usr/bin/sqlite3', - ], - 'backupper_options' => [ - 'default' => '--opt1 val1 -x -y -z --opt2 val2', + 'anonymization' => [ + 'connection_one' => [ + 'user' => [ + 'last_name' => 'fr-fr.firstname', + 'email' => [ + 'anonymizer' => 'email', + 'options' => [ + 'domain' => 'toto.com', + ], + ], + ], + ], ], - 'restorer_options' => [ - 'default' => '-abc -x val1 -y val2', + 'anonymization_files' => [ + 'connection_one' => 'connection_one.yaml', + 'connection_two' => 'connection_two.yaml', ], 'anonymizer_paths' => [ - '%kernel.project_dir%/src/Anonymization/Anonymizer', + './', ], - 'anonymization' => [ - 'yaml' => [ - 'default' => '%kernel.project_dir%/config/anonymization.yaml', + 'backup_binary' => '/path/to/dump', + 'backup_excluded_tables' => ['table1', 'table2'], + 'backup_expiration_age' => '2 minutes ago', + 'backup_options' => '--dump', + 'backup_timeout' => 135, + 'connections' => [ + 'connection_one' => [ + 'backup_binary' => '/path/to/dump/one', + 'backup_excluded_tables' => ['one1'], + 'backup_expiration_age' => '1 minutes ago', + 'backup_options' => '--dump-one', + 'backup_timeout' => 11, + 'restore_binary' => '/paht/to/restore/one', + 'restore_options' => '--restore-one', + 'restore_timeout' => 23, + 'storage_directory' => '/one/storage', + 'storage_filename_strategy' => 'one_strategy', + 'url' => null, ], ], - 'storage' => [ - 'root_dir' => '%kernel.project_dir%/var/db_tools', - 'filename_strategy' => [], - ], + 'default_connection' => 'connection_one', + 'restore_binary' => '/path/to/restore', + 'restore_options' => '--restore', + 'restore_timeout' => 357, + 'storage_directory' => '%kernel.project_dir%/var/db_tools', + 'storage_filename_strategy' => 'datetime', + 'workdir' => null, // '/path/to', ], - $result + $result, ); return $result; } - public function testConfigurationAlternative2(): array + public function testConfigurationConnectionsPartial(): array { $result = $this->processYamlConfiguration( - \dirname(__DIR__, 4) . '/Resources/config/packages/db_tools_alt2.yaml' + \dirname(__DIR__, 4) . '/Resources/config/packages/db_tools_connections_partial.yaml' ); - self::assertSame( + self::assertEquals( [ - 'storage' => [ - 'root_dir' => '%kernel.project_dir%/var/backup', - 'filename_strategy' => [ - 'connection_two' => 'app.db_tools.custom_filename_strategy', + 'anonymization' => [], + 'anonymization_files' => [], + 'anonymizer_paths' => [], + 'backup_binary' => '/path/to/dump', + 'backup_excluded_tables' => ['table1', 'table2'], + 'backup_expiration_age' => '2 minutes ago', + 'backup_options' => '--dump', + 'backup_timeout' => 135, + 'connections' => [ + 'connection_one' => [ + 'backup_binary' => null, + 'backup_excluded_tables' => ['one1'], + 'backup_expiration_age' => '1 minutes ago', + 'backup_options' => null, + //'backup_timeout' => null, + 'restore_binary' => null, + 'restore_options' => null, + 'restore_timeout' => 23, + 'storage_directory' => '/one/storage', + 'storage_filename_strategy' => 'one_strategy', + 'url' => null, ], ], - 'backup_expiration_age' => '6 months ago', - 'backup_timeout' => 1800, - 'restore_timeout' => 3200, - 'excluded_tables' => [ - 'connection_two' => ['table1', 'table2'], - ], - 'backupper_binaries' => [ - 'mariadb' => '/usr/bin/mariadb-dump', - 'mysql' => '/usr/bin/mysqldump', - 'postgresql' => '/usr/bin/pg_dump', - 'sqlite' => '/usr/bin/sqlite3', - ], - 'restorer_binaries' => [ - 'mariadb' => '/usr/bin/mariadb', - 'mysql' => '/usr/bin/mysql', - 'postgresql' => '/usr/bin/pg_restore', - 'sqlite' => '/usr/bin/sqlite3', - ], - 'backupper_options' => [ - 'connection_one' => '--opt1 val1 -x -y -z --opt2 val2', - ], - 'restorer_options' => [ - 'connection_one' => '-abc -x val1 -y val2', - 'connection_two' => '-a "Value 1" -bc -d val2 --end', - ], - 'anonymizer_paths' => [ - '%kernel.project_dir%/src/Anonymization/Anonymizer', + 'default_connection' => 'connection_one', + 'restore_binary' => '/path/to/restore', + 'restore_options' => '--restore', + 'restore_timeout' => 357, + 'storage_directory' => '%kernel.project_dir%/var/db_tools', + 'storage_filename_strategy' => 'datetime', + 'workdir' => null, // '/path/to', + ], + $result, + ); + + return $result; + } + + public function testDeprecatedV2(): void + { + $result = $this->processYamlConfiguration( + \dirname(__DIR__, 4) . '/Resources/config/packages/db_tools_deprecated_v2.yaml' + ); + + self::assertEquals( + [ + 'anonymization' => [], + 'anonymization_files' => [ + 'connection_one' => 'connection_one.yaml', + 'connection_two' => 'connection_two.yaml', ], - 'anonymization' => [ - 'yaml' => [ - 'connection_one' => '%kernel.project_dir%/config/anonymizations/connection_one.yaml', - 'connection_two' => '%kernel.project_dir%/config/anonymizations/connection_two.yaml', + 'anonymizer_paths' => [], + 'backup_binary' => null, + 'backup_excluded_tables' => [], + 'backup_options' => null, + 'connections' => [ + 'connection_one' => [ + 'backup_excluded_tables' => ['one1', 'one2'], + 'storage_filename_strategy' => 'some_strategy', + ], + 'connection_two' => [ + 'backup_excluded_tables' => ['two1', 'two2', 'two3'], ], ], + 'default_connection' => null, + 'restore_binary' => null, + 'restore_options' => null, + 'storage_directory' => '/foo/bar', + 'storage_filename_strategy' => null, + 'workdir' => null, ], - $result + $result, ); + } - return $result; + public function testDeprecatedV2Conflict(): void + { + self::expectExceptionMessage('Deprecated option "excluded_tables.connection_one" and actual option "connections.connection_one.backup_excluded_tables" are both defined, please fix your configuration.'); + + $this->processYamlConfiguration( + \dirname(__DIR__, 4) . '/Resources/config/packages/db_tools_deprecated_v2_conflict.yaml' + ); } public function testConfigurationBackupTimeoutInt(): void diff --git a/tests/Unit/Bridge/Symfony/DependencyInjection/DbToolsExtensionTest.php b/tests/Unit/Bridge/Symfony/DependencyInjection/DbToolsExtensionTest.php index 96351afc..cb86f8fe 100644 --- a/tests/Unit/Bridge/Symfony/DependencyInjection/DbToolsExtensionTest.php +++ b/tests/Unit/Bridge/Symfony/DependencyInjection/DbToolsExtensionTest.php @@ -4,17 +4,22 @@ namespace MakinaCorpus\DbToolsBundle\Tests\Unit\Bridge\Symfony\DependencyInjection; +use MakinaCorpus\DbToolsBundle\Anonymization\AnonymizatorFactory; use MakinaCorpus\DbToolsBundle\Bridge\Symfony\DependencyInjection\DbToolsExtension; -use PHPUnit\Framework\Attributes\DependsExternal; +use MakinaCorpus\DbToolsBundle\Configuration\ConfigurationRegistry; use PHPUnit\Framework\TestCase; +use PHPUnit\Framework\Attributes\After; +use PHPUnit\Framework\Attributes\DependsExternal; use Symfony\Component\DependencyInjection\ContainerBuilder; -use Symfony\Component\DependencyInjection\ParameterBag\ParameterBag; +use Symfony\Component\DependencyInjection\ParameterBag\EnvPlaceholderParameterBag; class DbToolsExtensionTest extends TestCase { + private array $originalEnv = []; + private function getContainer(array $parameters = [], array $bundles = []): ContainerBuilder { - $container = new ContainerBuilder(new ParameterBag($parameters + [ + $container = new ContainerBuilder(new EnvPlaceholderParameterBag($parameters + [ 'kernel.bundles' => $bundles, 'kernel.cache_dir' => \sys_get_temp_dir(), 'kernel.debug' => false, @@ -26,11 +31,6 @@ private function getContainer(array $parameters = [], array $bundles = []): Cont return $container; } - private function getMinimalConfig(): array - { - return []; - } - private function testExtension(array $config): void { $extension = new DbToolsExtension(); @@ -42,9 +42,157 @@ private function testExtension(array $config): void $container->compile(); } + #[After()] + protected function restoreEnv(): void + { + try { + foreach ($this->originalEnv as $name => $value) { + $_ENV[$name] = $value; + \putenv(\sprintf("%s=%s", $name, $value)); + } + } finally { + $this->originalEnv = []; + } + } + + private function putEnv(string $name, ?string $value): void + { + if (!\array_key_exists($name, $this->originalEnv)) { + $originalValue = \getenv($name); + $this->originalEnv[$name] = (false === $originalValue) ? '' : (string) $value; + } + $_ENV[$name] = (string) $value; + \putenv(\sprintf("%s=%s", $name, (string) $value)); + } + + private function putEnvAll(array $values): void + { + foreach ($values as $name => $value) { + $this->putEnv($name, $value); + } + } + + private function setAllDbToolsEnv(): void + { + $this->putEnvAll([ + 'DBTOOLS_BACKUP_BINARY' => '/usr/bin/fromenv-backup', + //'DBTOOLS_BACKUP_EXCLUDED_TABLES' => 'envtable1,envtable2', @todo + 'DBTOOLS_BACKUP_EXPIRATION_AGE' => '3 weeks', + 'DBTOOLS_BACKUP_OPTIONS' => '--from-env-backup', + 'DBTOOLS_BACKUP_TIMEOUT' => '666', + 'DBTOOLS_DEFAULT_CONNECTION' => 'fromenv-connection', + 'DBTOOLS_RESTORE_BINARY' => '/usr/bin/fromenv-restore', + 'DBTOOLS_RESTORE_OPTIONS' => '--from-env-restore', + 'DBTOOLS_RESTORE_TIMEOUT' => '999', + 'DBTOOLS_STORAGE_FILENAME_STRATEGY' => 'fromenv_strategy', + 'DBTOOLS_STORAGE_DIRECTORY' => '/fromenv/storage', + ]); + } + + #[DependsExternal(DbToolsConfigurationTest::class, 'testConfigurationFull')] + public function testEnvVarAreOverridenByConfiguration(array $config): void + { + $this->setAllDbToolsEnv(); + + $extension = new DbToolsExtension(); + $extension->load([$config], $container = $this->getContainer()); + $container->getDefinition('db_tools.configuration.registry')->setPublic(true); + $container->compile(true); + + $configRegistry = $container->get('db_tools.configuration.registry'); + \assert($configRegistry instanceof ConfigurationRegistry); + $defaultConfig = $configRegistry->getDefaultConfig(); + + // @todo missing excluded tables and default connection. + self::assertSame('/path/to/dump', $defaultConfig->getBackupBinary()); + self::assertSame('2 minutes ago', $defaultConfig->getBackupExpirationAge()); + self::assertSame('--dump', $defaultConfig->getBackupOptions()); + self::assertSame(135, $defaultConfig->getBackupTimeout()); + self::assertSame('/path/to/restore', $defaultConfig->getRestoreBinary()); + self::assertSame('--restore', $defaultConfig->getRestoreOptions()); + self::assertSame(357, $defaultConfig->getRestoreTimeout()); + self::assertSame('datetime', $defaultConfig->getStorageFilenameStrategy()); + self::assertSame(__DIR__ . '/var/db_tools', $defaultConfig->getStorageDirectory()); + } + + #[DependsExternal(DbToolsConfigurationTest::class, 'testConfigurationEmpty')] + public function testEnvVarArePropagated(array $config): void + { + $this->setAllDbToolsEnv(); + + $extension = new DbToolsExtension(); + $extension->load([$config], $container = $this->getContainer()); + $container->getDefinition('db_tools.configuration.registry')->setPublic(true); + $container->compile(true); + + $configRegistry = $container->get('db_tools.configuration.registry'); + \assert($configRegistry instanceof ConfigurationRegistry); + $defaultConfig = $configRegistry->getDefaultConfig(); + + // @todo missing excluded tables and default connection. + self::assertSame('/usr/bin/fromenv-backup', $defaultConfig->getBackupBinary()); + self::assertSame('3 weeks', $defaultConfig->getBackupExpirationAge()); + self::assertSame('--from-env-backup', $defaultConfig->getBackupOptions()); + self::assertSame(666, $defaultConfig->getBackupTimeout()); + self::assertSame('/usr/bin/fromenv-restore', $defaultConfig->getRestoreBinary()); + self::assertSame('--from-env-restore', $defaultConfig->getRestoreOptions()); + self::assertSame(999, $defaultConfig->getRestoreTimeout()); + self::assertSame('fromenv_strategy', $defaultConfig->getStorageFilenameStrategy()); + self::assertSame('/fromenv/storage', $defaultConfig->getStorageDirectory()); + } + + #[DependsExternal(DbToolsConfigurationTest::class, 'testConfigurationConnectionsPartial')] + public function testConnectionResolveParentForNonSetValues(array $config): void + { + $this->setAllDbToolsEnv(); + + $extension = new DbToolsExtension(); + $extension->load([$config], $container = $this->getContainer()); + $container->getDefinition('db_tools.configuration.registry')->setPublic(true); + $container->compile(true); + + $configRegistry = $container->get('db_tools.configuration.registry'); + \assert($configRegistry instanceof ConfigurationRegistry); + $defaultConfig = $configRegistry->getDefaultConfig(); + self::assertSame('/path/to/dump', $defaultConfig->getBackupBinary()); + + // Non existing will inherit everything from parent. + // This ensures we don't have any typo in Configuration class. + $nonExistingConnectionConfig = $configRegistry->getConnectionConfig('non_existing'); + // @todo missing excluded tables and default connection. + self::assertSame('/path/to/dump', $nonExistingConnectionConfig->getBackupBinary()); + self::assertSame('2 minutes ago', $nonExistingConnectionConfig->getBackupExpirationAge()); + self::assertSame('--dump', $nonExistingConnectionConfig->getBackupOptions()); + self::assertSame(135, $nonExistingConnectionConfig->getBackupTimeout()); + self::assertSame('/path/to/restore', $nonExistingConnectionConfig->getRestoreBinary()); + self::assertSame('--restore', $nonExistingConnectionConfig->getRestoreOptions()); + self::assertSame(357, $nonExistingConnectionConfig->getRestoreTimeout()); + self::assertSame('datetime', $nonExistingConnectionConfig->getStorageFilenameStrategy()); + self::assertSame(__DIR__ . '/var/db_tools', $nonExistingConnectionConfig->getStorageDirectory()); + + $connectionConfig = $configRegistry->getConnectionConfig('connection_one'); + // @todo missing excluded tables and default connection. + // Own values. + self::assertSame('1 minutes ago', $connectionConfig->getBackupExpirationAge()); + self::assertSame(23, $connectionConfig->getRestoreTimeout()); + self::assertSame('one_strategy', $connectionConfig->getStorageFilenameStrategy()); + self::assertSame('/one/storage', $connectionConfig->getStorageDirectory()); + // Inherited values. + self::assertSame('/path/to/dump', $connectionConfig->getBackupBinary()); + self::assertSame(135, $connectionConfig->getBackupTimeout()); + self::assertSame('--restore', $connectionConfig->getRestoreOptions()); + self::assertSame('/path/to/restore', $connectionConfig->getRestoreBinary()); + self::assertSame('--dump', $connectionConfig->getBackupOptions()); + } + + public function testConfigurationIsDefined(): void + { + self::markTestIncomplete(); + } + public function testExtensionRaiseErrorWhenUserPathDoesNotExist(): void { - $config = $this->getMinimalConfig(); + $config = []; $config['anonymizer_paths'] = ['/non_existing_path/']; $extension = new DbToolsExtension(); @@ -53,14 +201,44 @@ public function testExtensionRaiseErrorWhenUserPathDoesNotExist(): void $extension->load([$config], $this->getContainer()); } - public function testExtensionFromMinimalArrayConfig(): void + public function testExtensionWithEmptyConfig(): void { - $this->testExtension($this->getMinimalConfig()); + $this->testExtension([]); } - #[DependsExternal(DbToolsConfigurationTest::class, 'testConfigurationMinimal')] - public function testExtensionFromMinimalYamlConfig(array $config): void + #[DependsExternal(DbToolsConfigurationTest::class, 'testConfigurationEmpty')] + public function testExtensionWithMinimalConfig(array $config): void { $this->testExtension($config); } + + #[DependsExternal(DbToolsConfigurationTest::class, 'testConfigurationFull')] + public function testExtensionWithFullConfig(array $config): void + { + $this->testExtension($config); + } + + #[DependsExternal(DbToolsConfigurationTest::class, 'testConfigurationFull')] + public function testExtensionLoadsAnonymizationConfig(array $config): void + { + self::markTestSkipped(); + + /* + $extension = new DbToolsExtension(); + $extension->load([$config], $container = $this->getContainer()); + $container->getDefinition('db_tools.anonymization.anonymizator.factory')->setPublic(true); + $container->compile(true); + + $anonymizatorFactory = $container->get('db_tools.anonymization.anonymizator.factory'); + \assert($anonymizatorFactory instanceof AnonymizatorFactory); + + $anonymizator = $anonymizatorFactory->getOrCreate('connection_one'); + */ + } + + #[DependsExternal(DbToolsConfigurationTest::class, 'testConfigurationFull')] + public function testExtensionLoadsAnonymizationFilesConfig(array $config): void + { + self::markTestSkipped(); + } } diff --git a/tests/Unit/Storage/StorageTest.php b/tests/Unit/Storage/StorageTest.php index 0f64fb76..5ebf2a69 100644 --- a/tests/Unit/Storage/StorageTest.php +++ b/tests/Unit/Storage/StorageTest.php @@ -4,6 +4,8 @@ namespace MakinaCorpus\DbToolsBundle\Tests\Unit\Storage; +use MakinaCorpus\DbToolsBundle\Configuration\Configuration; +use MakinaCorpus\DbToolsBundle\Configuration\ConfigurationRegistry; use MakinaCorpus\DbToolsBundle\Storage\DefaultFilenameStrategy; use MakinaCorpus\DbToolsBundle\Storage\Storage; use MakinaCorpus\DbToolsBundle\Tests\Unit\Storage\Mock\OutOfRootFilenameStrategy; @@ -31,9 +33,14 @@ protected function tearDown(): void (new Filesystem())->remove(\sys_get_temp_dir() . '/db-tools-bundle-test'); } + protected function getConfigRegistry() + { + return new ConfigurationRegistry(new Configuration(storageDirectory: $this->rootDir, backupExpirationAge: 'now -6 month')); + } + public function testGenerateFilenameUsesDefaultWhenUnspecified(): void { - $storage = new Storage($this->rootDir, 'now -6 month'); + $storage = new Storage($this->getConfigRegistry()); self::assertMatchesRegularExpression( '@^' . \preg_quote($this->rootDir) . '/foo/\d{1,4}/\d{1,2}/foo-\d{6,14}\.some_ext$@', @@ -43,7 +50,7 @@ public function testGenerateFilenameUsesDefaultWhenUnspecified(): void public function testGenerateFilenameUsesStrategy(): void { - $storage = new Storage($this->rootDir, 'now -6 month', [ + $storage = new Storage($this->getConfigRegistry(), [ 'bar' => new TestFilenameStrategy(), ]); @@ -60,7 +67,7 @@ public function testGenerateFilenameUsesStrategy(): void public function testListBackups(): void { - $storage = new Storage($this->rootDir, 'now -6 month', [ + $storage = new Storage($this->getConfigRegistry(), [ 'outside' => new OutOfRootFilenameStrategy($this->outsideOfRootDir), 'inside' => new DefaultFilenameStrategy(), 'another' => new DefaultFilenameStrategy(),