From cd849d31700c44ec196453e4418dd6264daa3374 Mon Sep 17 00:00:00 2001 From: Paul Prescod Date: Tue, 24 Aug 2021 15:40:41 -0700 Subject: [PATCH 1/2] Support all relevant mapping features in load.yml --- snowfakery/cci_mapping_files/declaration_parser.py | 9 +++++++-- tests/cci/mapping_mixins.load.yml | 4 +++- tests/cci/test_declaration_parser.py | 8 +++++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/snowfakery/cci_mapping_files/declaration_parser.py b/snowfakery/cci_mapping_files/declaration_parser.py index 544ffd34..86f6dcb0 100644 --- a/snowfakery/cci_mapping_files/declaration_parser.py +++ b/snowfakery/cci_mapping_files/declaration_parser.py @@ -28,7 +28,7 @@ class AtomicDecl(T.NamedTuple): sf_object: str key: str - value: T.Union[T.List, str] + value: T.Union[T.List, str, date, int] priority: int merge_rule: T.Callable # what to do if two declarations for same val @@ -87,7 +87,12 @@ def case_normalizer(cls, val): return val def as_mapping(self): - rc = {"api": self.api, "bulk_mode": self.bulk_mode} + rc = { + "api": self.api, + "bulk_mode": self.bulk_mode, + "batch_size": self.batch_size, + "anchor_date": self.anchor_date, + } return {k: v for k, v in rc.items() if v is not None} diff --git a/tests/cci/mapping_mixins.load.yml b/tests/cci/mapping_mixins.load.yml index 2120b944..bddd48af 100644 --- a/tests/cci/mapping_mixins.load.yml +++ b/tests/cci/mapping_mixins.load.yml @@ -6,4 +6,6 @@ api: rest - sf_object: Account - api: smart + api: rest + batch_size: 100 + anchor_date: 2000-01-01 diff --git a/tests/cci/test_declaration_parser.py b/tests/cci/test_declaration_parser.py index 17d54ae3..0377d240 100644 --- a/tests/cci/test_declaration_parser.py +++ b/tests/cci/test_declaration_parser.py @@ -21,6 +21,8 @@ SObjectRuleDeclaration(sf_object="bar", priority="medium", api="smart"), SObjectRuleDeclaration(sf_object="bar", priority="medium", bulk_mode="serial"), SObjectRuleDeclaration(sf_object="bar", priority="high", bulk_mode="Parallel"), + SObjectRuleDeclaration(sf_object="foo", priority="low", batch_size=2000), + SObjectRuleDeclaration(sf_object="foo", priority="low", anchor_date="2000-01-01"), ] @@ -42,9 +44,9 @@ def test_priority_overrides(self): load_after=None, priority=None, api="bulk", - batch_size=None, + batch_size=2000, bulk_mode="serial", - anchor_date=None, + anchor_date="2000-01-01", ), } @@ -74,7 +76,7 @@ def test_cli__infers_load_file(self, tmpdir): "Insert Contact", "Insert Opportunity", ] - assert map_data["Insert Account"]["api"] == "smart" + assert map_data["Insert Account"]["api"] == "rest" def test_cli__explicit_file(self, tmpdir): sample_yaml = Path(__file__).parent / "mapping_mixins.recipe.yml" From dd8bd73014dd1520bf5e3e0c59dec340b50aa3b1 Mon Sep 17 00:00:00 2001 From: Paul Prescod Date: Mon, 27 Sep 2021 15:55:46 -0700 Subject: [PATCH 2/2] Update deps before Snowfakery 2.1 --- requirements/dev.txt | 57 ++++++++++++++++++++++--------------------- requirements/prod.txt | 13 +++++----- 2 files changed, 36 insertions(+), 34 deletions(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index caab59c5..0ad39e58 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -4,19 +4,17 @@ # # pip-compile --allow-unsafe requirements/dev.in # -appdirs==1.4.4 - # via black attrs==21.2.0 # via pytest backports.entry-points-selectable==1.1.0 # via virtualenv -black==21.7b0 +black==21.9b0 # via -r requirements/dev.in certifi==2021.5.30 # via requests -cfgv==3.3.0 +cfgv==3.3.1 # via pre-commit -charset-normalizer==2.0.3 +charset-normalizer==2.0.6 # via requests click==7.1.2 # via @@ -32,41 +30,41 @@ coverage==5.5 # pytest-cov coveralls==3.2.0 # via -r requirements/dev.in -distlib==0.3.2 +distlib==0.3.3 # via virtualenv docopt==0.6.2 # via coveralls faker-microservice==2.0.0 # via -r requirements/dev.in -faker==8.10.3 +faker==8.14.0 # via # -r requirements/prod.txt # faker-microservice -filelock==3.0.12 +filelock==3.1.0 # via # tox # virtualenv flake8==3.9.2 # via -r requirements/dev.in -ghp-import==2.0.1 +ghp-import==2.0.2 # via mkdocs -greenlet==1.1.0 +greenlet==1.1.1 # via # -r requirements/prod.txt # sqlalchemy gvgen==1.0 # via -r requirements/prod.txt -identify==2.2.11 +identify==2.2.15 # via pre-commit idna==3.2 # via # requests # yarl -immutables==0.15 +immutables==0.16 # via # -r requirements/prod.txt # contextvars -importlib-metadata==4.6.1 +importlib-metadata==4.8.1 # via # -r requirements/prod.txt # flake8 @@ -109,13 +107,15 @@ packaging==21.0 # tox pathspec==0.9.0 # via black -platformdirs==2.1.0 - # via virtualenv -pluggy==0.13.1 +platformdirs==2.4.0 + # via + # black + # virtualenv +pluggy==1.0.0 # via # pytest # tox -pre-commit==2.13.0 +pre-commit==2.15.0 # via -r requirements/dev.in py==1.10.0 # via @@ -133,7 +133,7 @@ pytest-cov==2.12.1 # via -r requirements/dev.in pytest-vcr==1.0.2 # via -r requirements/dev.in -pytest==6.2.4 +pytest==6.2.5 # via # -r requirements/dev.in # pytest-cov @@ -152,13 +152,13 @@ pyyaml==5.4.1 # pre-commit # pyyaml-env-tag # vcrpy -regex==2021.7.6 +regex==2021.9.24 # via black requests==2.26.0 # via # coveralls # responses -responses==0.13.3 +responses==0.14.0 # via -r requirements/dev.in six==1.16.0 # via @@ -168,7 +168,7 @@ six==1.16.0 # tox # vcrpy # virtualenv -sqlalchemy==1.4.22 +sqlalchemy==1.4.25 # via -r requirements/prod.txt text-unidecode==1.3 # via @@ -180,11 +180,11 @@ toml==0.10.2 # pytest # pytest-cov # tox -tomli==1.1.0 +tomli==1.2.1 # via black -tox-gh-actions==2.6.0 +tox-gh-actions==2.8.1 # via -r requirements/dev.in -tox==3.24.0 +tox==3.24.4 # via # -r requirements/dev.in # tox-gh-actions @@ -192,14 +192,15 @@ typed-ast==1.4.3 # via black typeguard==2.10.0 # via -r requirements/dev.in -typing-extensions==3.10.0.0 +typing-extensions==3.10.0.2 # via # -r requirements/prod.txt # black + # immutables # importlib-metadata # pydantic # yarl -urllib3==1.26.6 +urllib3==1.26.7 # via # requests # responses @@ -207,11 +208,11 @@ vcrpy==4.1.1 # via # -r requirements/dev.in # pytest-vcr -virtualenv==20.6.0 +virtualenv==20.8.1 # via # pre-commit # tox -watchdog==2.1.3 +watchdog==2.1.5 # via mkdocs wrapt==1.12.1 # via vcrpy diff --git a/requirements/prod.txt b/requirements/prod.txt index f517b071..83b3baeb 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -8,15 +8,15 @@ click==7.1.2 # via -r requirements/prod.in contextvars==2.4 # via -r requirements/prod.in -faker==8.10.3 +faker==8.14.0 # via -r requirements/prod.in -greenlet==1.1.0 +greenlet==1.1.1 # via sqlalchemy gvgen==1.0 # via -r requirements/prod.in -immutables==0.15 +immutables==0.16 # via contextvars -importlib-metadata==4.6.1 +importlib-metadata==4.8.1 # via sqlalchemy jinja2==2.11.3 # via -r requirements/prod.in @@ -32,13 +32,14 @@ pyyaml==5.4.1 # via -r requirements/prod.in six==1.16.0 # via python-dateutil -sqlalchemy==1.4.22 +sqlalchemy==1.4.25 # via -r requirements/prod.in text-unidecode==1.3 # via faker -typing-extensions==3.10.0.0 +typing-extensions==3.10.0.2 # via # -r requirements/prod.in + # immutables # importlib-metadata # pydantic zipp==3.5.0