From af6eded6a108ff4344b5eea080718fad755c806f Mon Sep 17 00:00:00 2001 From: James Murty Date: Mon, 6 Jul 2020 10:56:43 +1000 Subject: [PATCH 01/13] Add script to reset local development environment --- ixc_django_docker/bin/dev-reset-environment.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100755 ixc_django_docker/bin/dev-reset-environment.sh diff --git a/ixc_django_docker/bin/dev-reset-environment.sh b/ixc_django_docker/bin/dev-reset-environment.sh new file mode 100755 index 0000000..d84aa78 --- /dev/null +++ b/ixc_django_docker/bin/dev-reset-environment.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# Reset a local development environment + +set -e + +if [[ -z "$PROJECT_DIR" ]]; then + >&2 echo "ERROR: Missing environment variable: PROJECT_DIR" + exit 1 +fi + +rm -rf "$DIR/var" + +find . -name "*.md5" -delete From 85a95dd6190e30758eb0c1aecea4779738ae949b Mon Sep 17 00:00:00 2001 From: James Murty Date: Mon, 6 Jul 2020 12:16:37 +1000 Subject: [PATCH 02/13] IC's storage module requires requests --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index bd9d0db..40f05b6 100644 --- a/setup.py +++ b/setup.py @@ -93,6 +93,7 @@ 'storages': [ 'boto3', 'django-storages', + 'requests', ], 'whitenoise': [ 'ixc-whitenoise', From d6a135e2db23422f6803d4007bddc3d7412ab3da Mon Sep 17 00:00:00 2001 From: James Murty Date: Mon, 6 Jul 2020 12:16:57 +1000 Subject: [PATCH 03/13] Use consistent slash-after-directory --- project_template/.dockerignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project_template/.dockerignore b/project_template/.dockerignore index 984b92f..7499750 100644 --- a/project_template/.dockerignore +++ b/project_template/.dockerignore @@ -20,7 +20,7 @@ project_settings_local.py src/ static_root/ var/ -venv +venv/ # Docker ignores. *.secret From 7d35c584eba33f4e9f96303ce07ca5cd52388192 Mon Sep 17 00:00:00 2001 From: James Murty Date: Mon, 6 Jul 2020 12:22:31 +1000 Subject: [PATCH 04/13] Example to set up Bandit email hijacking by default --- project_template/.env.local.sample | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/project_template/.env.local.sample b/project_template/.env.local.sample index 92f8967..2cbabf2 100644 --- a/project_template/.env.local.sample +++ b/project_template/.env.local.sample @@ -2,3 +2,7 @@ export DOTENV='develop' export TRANSCRYPT_PASSWORD='' # Get from 1Password # export WITHOUT_COVERAGE=1 + +# Safely send real emails; Bandit prevents sending to arbitrary addresses +export BANDIT_EMAIL='YOURNAME@interaction.net.au' +export BANDIT_WHITELIST='interaction.net.au,user_abc@client.org.au' From f165ecff0903f3f70f70afa7898fea65fc4590c1 Mon Sep 17 00:00:00 2001 From: James Murty Date: Mon, 6 Jul 2020 12:30:28 +1000 Subject: [PATCH 05/13] Need npm-run-all to get the run-s command --- project_template/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/project_template/package.json b/project_template/package.json index c7c0bbe..499e3c4 100644 --- a/project_template/package.json +++ b/project_template/package.json @@ -2,6 +2,7 @@ "name": "project_template", "dependencies": { "bower": "^1.8.2", + "npm-run-all": "^4.1.5", "sass": "^1.9.0" }, "private": true, From dfcffbeb95eb02577220c945948ed086186af1e4 Mon Sep 17 00:00:00 2001 From: James Murty Date: Tue, 7 Jul 2020 16:04:07 +1000 Subject: [PATCH 06/13] Codefresh pushes untested Docker image to make it cacheable Push the built but pre-test Docker image in parallel with the tests run so that the image will be available for fetching from Docker Hub if necessary, e.g. if you re-run the Codefresh run, or want to fetch and debug a failing image locally. This is mainly motivated by the fact that Codefresh no longer provides its own repository for images, which it used to, and thus no longer caches every built image including those that failed the `run_tests` step. --- project_template/codefresh.yml | 83 +++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 36 deletions(-) diff --git a/project_template/codefresh.yml b/project_template/codefresh.yml index 02c3b72..53842c1 100644 --- a/project_template/codefresh.yml +++ b/project_template/codefresh.yml @@ -9,48 +9,59 @@ steps: type: build image_name: interaction/project_template - run_tests: - type: composition - composition: - version: '2' + # Run tests and push built image in parallel + in_parallel: + type: parallel + steps: - services: - django: - depends_on: - # - elasticsearch - - postgres - - redis - environment: - DOTENV: test - # ELASTICSEARCH_ADDRESS: elasticsearch:9200 - PGHOST: postgres - PGUSER: postgres - REDIS_ADDRESS: redis:6379 - image: '${{build_image}}' + # Push UNTESTED image to improve caching (not to be used directly) + push_untested_image: + candidate: '${{build_image}}' + type: push + tag: 'untested-${{CF_REVISION}}' - # elasticsearch: - # image: interaction/elasticsearch-icu:7-alpine + run_tests: + type: composition + composition: + version: '2' - postgres: - environment: - POSTGRES_HOST_AUTH_METHOD: trust - image: postgres:12.2-alpine + services: + django: + depends_on: + # - elasticsearch + - postgres + - redis + environment: + DOTENV: test + # ELASTICSEARCH_ADDRESS: elasticsearch:9200 + PGHOST: postgres + PGUSER: postgres + REDIS_ADDRESS: redis:6379 + image: '${{build_image}}' - redis: - image: redis:6-alpine + # elasticsearch: + # image: interaction/elasticsearch-icu:7-alpine - volumes: - # elasticsearch-data: - postgres-data: - redis-data: + postgres: + environment: + POSTGRES_HOST_AUTH_METHOD: trust + image: postgres:12.2-alpine - composition_candidates: - django: - command: runtests.sh --failfast . - when: - condition: - all: - skip_tests_variable: lower('${{CF_SKIP_TESTS}}') != 'true' + redis: + image: redis:6-alpine + + volumes: + # elasticsearch-data: + postgres-data: + redis-data: + + composition_candidates: + django: + command: runtests.sh --failfast . + when: + condition: + all: + skip_tests_variable: lower('${{CF_SKIP_TESTS}}') != 'true' push_images: candidate: '${{build_image}}' From a39a5c47bb9b9080820082f4b0e074a1b7c1aa53 Mon Sep 17 00:00:00 2001 From: James Murty Date: Tue, 7 Jul 2020 16:06:01 +1000 Subject: [PATCH 07/13] Add recommended lib version constraints for Django 1.8 These are the constraints for ixc-django-docker's dependencies that worked for the RFDS site that uses Django 1.8, when re-creating that project's pinned requirements.txt file from scratch. --- project_template/requirements.in | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/project_template/requirements.in b/project_template/requirements.in index bdb8108..cdee59f 100644 --- a/project_template/requirements.in +++ b/project_template/requirements.in @@ -1,2 +1,24 @@ # -e ..[celery,celery4,celery-email,compressor,datadog,debug-toolbar,email-bandit,extensions,logentries,master-password,newrelic,nose,post-office,postgres,pydevd,sentry,storages,whitenoise] -e git+https://github.com/ixc/ixc-django-docker.git@master#egg=ixc-django-docker[celery,celery4,celery-email,compressor,datadog,debug-toolbar,email-bandit,extensions,logentries,master-password,newrelic,nose,post-office,postgres,pydevd,sentry,storages,whitenoise] + + +############################################################################## +# Django<1.11 compatibility recommendations + +# -e git+https://github.com/ixc/ixc-whitenoise.git@9812e7b3caa62a438c9332653c2b297856d6f104#egg=ixc-whitenoise # Compatibilty with older Django +# celery<4.3 # Version 4.3 requires Django>=1.11 +# django-appconf<1.0.4 # Version 1.0.4 should work with Django 1.8 but doesn't +# django-celery-beat<1.6 # Version 1.6 requires Django>=1.11.17 +# django-celery-email<3 # Version 3.0.0 requires Django>1 +# django-celery-results<1.1 # Version 1.1 requires celery>=4.3 +# django-compressor<2.2 # Version 2.3 requires Django>1.11, but 2.2 doesn't work +# django-debug-toolbar<1.10 # Version 1.10 requires Django>=1.11 +# django-extensions<2.1.1 # Version 2.1.1 requires Django>=1.11 +# django-ipware<3 # Version 3 requires Python>=3.5 +# django-post-office<3.3 # Version 3.3.0 requires Django>=1.11 +# django-redis<4.9 # Version 4.9.0 requires Django>=1.11 +# django-storages<1.6.6 # Version 1.6.6 requires Django>=1.11 +# django-timezone-field<3.1 # Version 3.1 requires Django>=1.11 +# jsonfield<2.1 # Version 2.1 requires Django>=1.11 + +############################################################################## From 7ee6689d66125fdd514ac3a08922c476dcb4bd57 Mon Sep 17 00:00:00 2001 From: James Murty Date: Tue, 7 Jul 2020 16:06:26 +1000 Subject: [PATCH 08/13] Add example library dependencies for sites that use Thumbor --- project_template/requirements.in | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/project_template/requirements.in b/project_template/requirements.in index cdee59f..0b572ce 100644 --- a/project_template/requirements.in +++ b/project_template/requirements.in @@ -22,3 +22,12 @@ # jsonfield<2.1 # Version 2.1 requires Django>=1.11 ############################################################################## + + +############################################################################## +# Use Thumbor for thumbnails (latest versions of IC forks of Thumbor libs) + +# -e git+https://github.com/ixc/django-thumbor.git@481cf39536b384369b2bc31da2af2bee34a4c617#egg=django-thumbor +# -e git+https://github.com/ixc/libthumbor.git@d5f44bbd6cd7c19f61f47d941213d1de52209521#egg=libthumbor + +############################################################################## From b2444e3b958d242a733f1555e453653f7ac7a212 Mon Sep 17 00:00:00 2001 From: James Murty Date: Tue, 7 Jul 2020 16:52:55 +1000 Subject: [PATCH 09/13] Make DOTENV available as a default setting DOTENV is useful in many places, such as setting unique names for things like databases or ElasticSearch indexes etc. Make it easily available as a setting that is guaranteed to be present, and given a clear default value if not, instead of requiring projects to hunt for and handle the 'DOTENV' environment variable. --- ixc_django_docker/settings/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ixc_django_docker/settings/base.py b/ixc_django_docker/settings/base.py index 2ce8225..13afc04 100644 --- a/ixc_django_docker/settings/base.py +++ b/ixc_django_docker/settings/base.py @@ -20,6 +20,9 @@ from django.utils.six import text_type +# Store DOTENV from environment as a setting +DOTENV = os.environ.get('DOTENV', 'dotenv-not-set') + # Get project directory from environment. This MUST already be defined. # Copied from __init__.py I'm not sure why it's needed here as well PROJECT_DIR = os.environ['PROJECT_DIR'].rstrip('/') From 37bc343a9acff80e7b92c00233c779e81dcea4da Mon Sep 17 00:00:00 2001 From: James Murty Date: Wed, 8 Jul 2020 10:38:18 +1000 Subject: [PATCH 10/13] Clearing out src/ as well can help fully reset / un-break a dev environment --- docs/how-to-upgrade.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to-upgrade.md b/docs/how-to-upgrade.md index 07c5127..e5f166e 100644 --- a/docs/how-to-upgrade.md +++ b/docs/how-to-upgrade.md @@ -185,7 +185,7 @@ Test your changes: Things to watch out for: -- Run `rm -rf var` and `find . -name "*.md5" -delete` to reset the environment and start over. +- Run `rm -rf var/ src/` and `find . -name "*.md5" -delete` to reset the environment and start over. - Upgrading Node.js can require that we also upgrade dependencies in `package.json`. If you see any npm errors during setup: From 3876fdd2ec16ee7a9367c96f8c90f361fa0a6fc6 Mon Sep 17 00:00:00 2001 From: James Murty Date: Wed, 8 Jul 2020 14:25:56 +1000 Subject: [PATCH 11/13] Remove ref to project-specific admin_tools from default URLs The `admin_tools` app comes from django-admin-tools which isn't referenced anywhere else in ixc-django-docker, so this was probably added to the default URLs by mistake. --- ixc_django_docker/urls.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ixc_django_docker/urls.py b/ixc_django_docker/urls.py index 7aefa2a..9fc7493 100644 --- a/ixc_django_docker/urls.py +++ b/ixc_django_docker/urls.py @@ -19,7 +19,6 @@ if 'django.contrib.admin' in settings.INSTALLED_APPS: urlpatterns += [ url(r'^admin/doc/', include('django.contrib.admindocs.urls')), - url(r'^admin/util/tools/', include('admin_tools.urls')), url(r'^admin/', include(admin.site.urls)), ] From 36698c230c73152c19bd8b91bff74928077c9a91 Mon Sep 17 00:00:00 2001 From: James Murty Date: Wed, 8 Jul 2020 14:29:27 +1000 Subject: [PATCH 12/13] Add setting ADMIN_URL to customise location of site admin For RFDS the site admin is under /rfds-admin/ not /admin/. The new ADMIN_URL setting makes the admin URL prefix easily configurable in downstream projects without needing to mess with URL patterns. --- ixc_django_docker/settings/base.py | 3 +++ ixc_django_docker/urls.py | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ixc_django_docker/settings/base.py b/ixc_django_docker/settings/base.py index 13afc04..2b8c9f4 100644 --- a/ixc_django_docker/settings/base.py +++ b/ixc_django_docker/settings/base.py @@ -139,6 +139,9 @@ MEDIA_ROOT = os.path.join(VAR_DIR, 'media_root') MEDIA_URL = '/media/' +# Prefix for admin URL paths, see `ixc_django_docker.urls` +ADMIN_URL = '/admin/' + # # HTTPS # diff --git a/ixc_django_docker/urls.py b/ixc_django_docker/urls.py index 9fc7493..88e8fd2 100644 --- a/ixc_django_docker/urls.py +++ b/ixc_django_docker/urls.py @@ -17,9 +17,10 @@ # Django Admin. if 'django.contrib.admin' in settings.INSTALLED_APPS: + _prefix = settings.ADMIN_URL.strip("/") urlpatterns += [ - url(r'^admin/doc/', include('django.contrib.admindocs.urls')), - url(r'^admin/', include(admin.site.urls)), + url(r'^%s/doc/' % _prefix, include('django.contrib.admindocs.urls')), + url(r'^%s/' % _prefix, include(admin.site.urls)), ] # Django Auth. From 3812b4e10655cdd6f6c282d853717d4350799212 Mon Sep 17 00:00:00 2001 From: James Murty Date: Mon, 13 Jul 2020 17:19:01 +1000 Subject: [PATCH 13/13] Remove redundant `npm run build` step This step is redundant in setup.sh because it gets run as part of the recommended Dockerfile when building an image. Before this change static files etc would get generated by the Codefresh build, then again slowly and pointlessly when you run the setup service. So either this command should be removed altogether from this script, or maybe be adjusted to run only in a local development context? --- ixc_django_docker/bin/setup.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ixc_django_docker/bin/setup.sh b/ixc_django_docker/bin/setup.sh index b085a97..9fc4f63 100755 --- a/ixc_django_docker/bin/setup.sh +++ b/ixc_django_docker/bin/setup.sh @@ -32,10 +32,6 @@ setup-postgres.sh # Apply migrations. migrate.sh "$PROJECT_DIR/var" -# Run build script. -echo "Executing: npm run ${SETUP_NPM_RUN:-build}..." -npm run "${SETUP_NPM_RUN:-build}" --if-present - # Save git commit. echo "$(git rev-parse HEAD)" > "$PROJECT_DIR/var/setup-git-commit.txt" echo "Updated '$PROJECT_DIR/var/setup-git-commit.txt' ($(cat $PROJECT_DIR/var/setup-git-commit.txt))"