Merge branch 'release/v17.2.23'

.gitignore

@@ -14,3 +14,9 @@ include
 man
 pip-selfcheck.json
 log
+portal/test_uploads
+
+.idea
+.settings/
+.pydevproject
+.project

.travis.yml

@@ -1,48 +1,140 @@
 language: python
 python: "2.7"
-
+sudo: false
 cache: pip
 addons:
   postgresql: "9.4"
+
 services:
   - postgresql
   - redis-server
 env:
+  matrix:
+    - TOXENV=py27
   global:
     - SQLALCHEMY_DATABASE_URI='postgresql://postgres:@localhost/portal_unit_tests'
     - LOG_FOLDER='/tmp/shared_service_log'
-    - PERSISTENCE_FILE='https://raw.githubusercontent.com/uwcirg/TrueNTH-USA-site-config/develop/site_persistence_file.json'
+    - PACKAGE_NAME=${PACKAGE_NAME:-true_nth_usa_portal}
+    - IMAGE_NAME=${IMAGE_NAME:-portal_web}
+    - BINTRAY_DEB_REPO=${BINTRAY_DEB_REPO:-true_nth}
+    - BINTRAY_DOCKER_REPO=${BINTRAY_DOCKER_REPO:-portal}
+    - BINTRAY_USERNAME=${BINTRAY_USERNAME:-uwcirg}
 
 matrix:
   include:
     - env: TOXENV=docs
-    - env:
-        - TOXENV=ui
-
-        # _SAUCE_USERNAME
-        - secure: "Hc5NUBDw0sUdXVTn0s8NqcmabMYZiMrpq3x5+Nytze4JKRxk0kmhyt IurC/C3A6bDM622ZFVtTCAs8k4Cg4GBfuGrdC/EaVaolw7trhWY9TUEuwWSPlmdTs ChPDQpGxXngxEA7ij2D6BOUR97l3UrXtMQ24syVHJl8MiLoZelNKSnQCs8yeNMw+j iLTvCyAXTl4hfoUiRidVxUlCTcSgJ4xajBDPEgAIm82Oo4qnCz7PSt/9GSG7cC5qH +KllMGbn/a8kTM+kJxiWA1QLQ7+DNo0TiYX1dMyZQPhPE/8H7EfLLvIuEAAvKNg3Y VxfpZYNDOKadP28HjSUqexbh75KBt8KZGweymLKPK/bt8zwAtWufEqW2iESlOfURz jKZ451oaBmCEpaFbbsxc6lnGQxnKB4/YTx6jClXTjii2dqUUw/lUIYgFXfnRYoyfo HGrl+AzZiVV4knJfQFHNT7rZoKfzgqzLUlkMYQu9sLKzfSb4ZT3MSFUnATfHmczmC 9JnX0aCtSj6V6ab6+PovFyFHRKhmn/zVAxUq0hFVdqZQj7A6nIU+vx8fpa42BRjA9 muaz3MwYIr5dAAlF1wPh3XRWx0mRW6YK4WYQ3HaV1nol9wGAnnTM0l949CN0u3IS6 WorfdmGuccFT0DT39qNyu5tnJ2W2hab7Dg2fJ+oabhEQ="
-
-        # _SAUCE_ACCESS_KEY
-        - secure: "jj5q57S8VzAHlNUlSrKtRdoCbFKFig75CFA8AxAQ0lyoMi9VOzwoNQ S3A50I0OYONhqiMWOeyxaBX+eS6y50kYoholJ8yy2DEzaYAJKGGrCh/S6YsGqBo9w jqODynf8c9uDeq4uyNqm3xrOuOBhYUWlmzczILkegw9cvBmKjMM8KK0OxwoEZlQC+ XqZMtUj0oxjiqyK5BiSDBt1TM8pLbZDUup0JTXzuxrRKjZnmw/LtMRqE5P64JeM7d 7qvcYwFrAMydUZSCIO9hguRjmL+kX2cl7Z0lTk5fAtcsroeDy63ClA2v/I90D4Bq2 v90jujU38P7ukJFLR4ji8HRKTllIJ1ouYoR4gzrWLQt7vXL7tHfByaCf4jFOqRaSd dEzj7xn4dXA/CbmbsNQSb+vFMVJMdND3RQDNYXcnGe/jlrn01UdHkouVui0SGtdl2 CHd8GfynkWhlw8XnZ6YZwdM8TVkYrFJPyD2hh7RyOzjytQb6gGdPVbztfEbgAjwXe af9/+Hi/aNdTlvF+KOZYr1Jkf5L2XTUxQhsNt+M2VMLzX03VOivlXZbRKC72b+hGv q93ENiMOt4uhT4d74KuTeXU19QelUWD5tYUiWEsIFvzz/g1+xVShU4DcNLKRIoYZY 1H4w95+QFRBZ9oXHRO7auua4QneIsFBq1ghQ/E7jfVOs="
+      services: []
+      addons: {}
+    - env: TOXENV=ui
       addons:
         postgresql: "9.4"
         sauce_connect: true
+    - env: TOXENV=build
+      services:
+        - docker
+  # Build artifacts only (no testing)
+  allow_failures:
+    - env: TOXENV=build
+      services:
+        - docker
 
 before_install:
-  # Set TOXENV default (if not overridden by matrix environmental variables)
-  - if [[ ! -v TOXENV ]]; then export TOXENV=py27;fi
+  # Set Github env vars
+  - IFS=/ read GH_USERNAME GH_REPO <<< "$TRAVIS_REPO_SLUG"
+  - export GH_USERNAME GH_REPO
+  - export SHORT_HASH=${TRAVIS_COMMIT::8}
+
+  # Only deploy from develop and master branches, and not previously configured
+  - if [[ ! -v DEPLOY && ("$TRAVIS_BRANCH" = "master" || "$TRAVIS_BRANCH" = "develop") ]]; then export DEPLOY=true;fi
 
-  # Allow environmental variables configured per-repo on TravisCI website to override .travis.yml defaults
-  - if [[ ! -v SAUCE_USERNAME ]]; then export SAUCE_USERNAME=$_SAUCE_USERNAME;fi
-  - if [[ ! -v SAUCE_ACCESS_KEY ]]; then export SAUCE_ACCESS_KEY=$_SAUCE_ACCESS_KEY;fi
+  # Set TOXENV default (if not already set by matrix environmental variables)
+  - if [[ ! -v TOXENV ]]; then export TOXENV=py27;fi
 
+  # Fetch all remote branches instead of just the currently checked out one
+  - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
+  # setuptools_scm module needs the git history since the last master branch to correctly calculate the number of revisions since then (ie devNNN in 16.12.8.3.dev274+ngd08a23c)
+  - git fetch --unshallow
 
-  - psql -c 'create database portal_unit_tests owner postgres;' -U postgres
+  - psql --command 'create database portal_unit_tests owner postgres;' --username postgres
 
-  - pip install --upgrade pip
+  - pip install --upgrade pip setuptools
+  - export PORTAL_VERSION="$(python setup.py --version)"
 
 install:
   - pip install tox
 
 script:
   - tox
+
+after_success:
+  # Only run build steps in corresponding tox env
+  - if [[ "$TOXENV" != "build" ]]; then travis_terminate 0;fi
+
+  # Set docker tag based on branch
+  - declare -A TAGS=(["develop"]="latest" ["master"]="stable"); export TAG=${TAGS["${TRAVIS_BRANCH}"]:-${TRAVIS_BRANCH//\//_}}
+
+  - docker pull "uwcirg-docker-true_nth.bintray.io/deb_builder:latest"
+  - docker volume create --name debian-repo
+  # Build artifacts from current repo and branch (else "develop")
+  - docker run --volume debian-repo:/tmp/artifacts --env REPO_SLUG=$TRAVIS_REPO_SLUG --env BRANCH=${TRAVIS_BRANCH:-develop} "uwcirg-docker-true_nth.bintray.io/deb_builder:latest"
+
+  # Copy artifacts off volume mounted on previous container
+  - docker cp $(docker ps --quiet --latest --filter ancestor="uwcirg-docker-true_nth.bintray.io/deb_builder:latest"):/tmp/artifacts /tmp
+  # Copy artifacts to current directory to allow local installation
+  - cp --recursive --verbose /tmp/artifacts artifacts
+  - docker build --file docker/Dockerfile --tag "$IMAGE_NAME:$SHORT_HASH" --tag "$IMAGE_NAME:$TAG" --tag "$IMAGE_NAME:$TRAVIS_BUILD_NUMBER" .
+
+before_deploy:
+  - export DOCKER_CLOUD_USERNAME=${DOCKER_CLOUD_USERNAME:-${GH_USERNAME//-}}
+  - docker login --username "$BINTRAY_USERNAME" --password "$BINTRAY_API_KEY" "$GH_USERNAME-docker-$BINTRAY_DOCKER_REPO.bintray.io"
+  - if [[ -v DOCKER_CLOUD_API_KEY ]]; then docker login --username "$DOCKER_CLOUD_USERNAME" --password "$DOCKER_CLOUD_API_KEY" https://index.docker.io/v1/;fi
+
+  # Build bintray descriptor file dynamically
+  - >
+    printf '
+    {
+        "package": {
+            "name": "$PACKAGE_NAME",
+            "repo": "$BINTRAY_DEB_REPO",
+            "subject": "$GH_USERNAME",
+            "public_download_numbers": false,
+            "public_stats": false
+        },
+        "version": {
+            "name": "$PORTAL_VERSION",
+            "gpgSign": true
+        },
+        "files": [
+            {
+                "includePattern": "artifacts/(.+\\.deb)$",
+                "uploadPattern": "artifacts/$1",
+                "matrixParams": {
+                    "deb_distribution": "stable",
+                    "deb_component": "main",
+                    "deb_architecture": "amd64"
+                }
+            }
+        ],
+        "publish": true
+    }
+    ' | envsubst | tee "${TRAVIS_BUILD_DIR}/bintray.json"
+
+# Only deploy from default virtual env TravisCI job
+deploy:
+  on:
+    all_branches: true
+    condition:
+      - $TOXENV = build
+      - $DEPLOY = true
+      - -v BINTRAY_API_KEY
+  provider: bintray
+  file: "bintray.json"
+  user: $BINTRAY_USERNAME
+  key: $BINTRAY_API_KEY
+
+after_deploy:
+  - docker tag $IMAGE_NAME:latest "$GH_USERNAME-docker-$BINTRAY_DOCKER_REPO.bintray.io/$IMAGE_NAME:latest"
+  - docker push "$GH_USERNAME-docker-$BINTRAY_DOCKER_REPO.bintray.io/$IMAGE_NAME:latest"
+
+  - docker tag $IMAGE_NAME:latest "$DOCKER_CLOUD_USERNAME/$IMAGE_NAME:latest"
+  - if [[ -v DOCKER_CLOUD_API_KEY ]]; then docker push "docker.io/$DOCKER_CLOUD_USERNAME/$IMAGE_NAME";fi

bin/deploy.sh

@@ -106,15 +106,14 @@ fi
 
 # DB Changes
 if [[
-    ($FORCE || ( -n $(git diff $old_head $new_head -- ${GIT_WORK_TREE}/migrations) && $? -eq 0 )) &&
+    ($FORCE || ( -n $(git diff $old_head $new_head -- ${GIT_WORK_TREE}/portal/migrations) && $? -eq 0 )) &&
     -z $INIT
 ]]; then
     activate_once
 
     if [[ $VERBOSE ]]; then
         echo "Running database migrations"
     fi
-    cd "${GIT_WORK_TREE}"
     python "${GIT_WORK_TREE}/manage.py" db upgrade
 fi
 

bin/testconnection.py

@@ -1,11 +1,11 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 import psycopg2
 import sys
 import os
 
 def parse_connection_uri():
     here, _ = os.path.split(__file__)
-    with open(os.path.join(here, '../portal/application.cfg'), 'r') as fh:
+    with open(os.path.join(here, '../instance/application.cfg'), 'r') as fh:
         conn_strings = [l for l in fh.readlines() if
                         l.startswith('SQLALCHEMY_DATABASE_URI')]
 

debian/changelog

@@ -0,0 +1,6 @@
+portal (VERSION) unstable; urgency=low
+
+  * Changelog line
+
+
+ -- CIRG, University of Washington <[email protected]>  TIMESTAMP

debian/compat

@@ -0,0 +1 @@
+9

debian/control

@@ -0,0 +1,13 @@
+Source: portal
+Section: python
+Priority: extra
+Maintainer: CIRG, University of Washington <[email protected]>
+Build-Depends: debhelper (>= 9), python, dh-virtualenv, libc6-i386, fakeroot
+Standards-Version: 3.9.5
+
+Package: portal
+Architecture: any
+Pre-Depends: dpkg (>= 1.16.1), python2.7-minimal | python2.6-minimal, ${misc:Pre-Depends}
+Depends: ${python:Depends}, ${misc:Depends}
+Description: TrueNTH Shared Services
+ TrueNTH Shared Services

debian/portal.triggers

@@ -0,0 +1,7 @@
+# Register interest in Python interpreter changes (Python 2 for now); and
+# don't make the Python package dependent on the virtualenv package
+# processing (noawait)
+interest-noawait /usr/bin/python2.7
+
+# Also provide a symbolic trigger for all dh-virtualenv packages
+interest dh-virtualenv-interpreter-update

debian/rules

@@ -0,0 +1,35 @@
+#!/usr/bin/make -f
+
+DH_VIRTUALENV_ARGS = --preinstall='pip>=8' --preinstall='setuptools>=12'
+
+
+distclean:
+	rm -Rf build debian/portal/ env
+
+
+override_dh_virtualenv:
+	# Remove self-referential requirement
+	# Otherwise a .egg-link file is created instead of actually installing portal module in site-packages/
+	sed -i '/-e ./d' requirements.txt
+
+	dh_virtualenv $(DH_VIRTUALENV_ARGS)
+
+override_dh_builddeb:
+# Manually fix paths erroneously pointing to build environment
+# Todo: move to more appropriate step
+
+	find \
+		debian/portal/opt/venvs/portal/lib/ \
+		-name easy-install.pth \
+		-exec \
+			sed \
+				--in-place \
+				--expression 's|$(PWD)/debian/portal/opt/venvs/|/opt/venvs/|g' \
+				{} \; \
+		-print
+
+	dh_builddeb
+
+
+%:
+	dh $@ --with python-virtualenv

docker/Dockerfile

@@ -0,0 +1,83 @@
+FROM debian:jessie
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN \
+    apt-get update && \
+    apt-get dist-upgrade --yes && \
+    apt-get install --yes --force-yes --no-install-recommends \
+        apache2 \
+        libapache2-mod-wsgi \
+        libpq5 \
+        python2.7 && \
+    apt-get clean
+
+RUN printf '\n\
+<VirtualHost *:80>\n\
+  ServerName ${APACHE_SERVER_NAME}\n\
+  LogLevel info\n\
+  ErrorLog /dev/stderr\n\
+  CustomLog /dev/stdout combined\n\
+  WSGIPassAuthorization On\n\
+  WSGIDaemonProcess stg_cs user=${APACHE_RUN_USER} group=${APACHE_RUN_GROUP} threads=5\n\
+  WSGIScriptAlias / /var/www/portal.wsgi\n\
+  <Directory /var/www/>\n\
+    WSGIProcessGroup stg_cs\n\
+    WSGIApplicationGroup %%{GLOBAL}\n\
+    Require all granted\n\
+  </Directory>\n\
+</VirtualHost>\n\
+\n'\
+> /etc/apache2/sites-available/portal.conf
+
+RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 379CE192D401AB61
+
+ARG debian_repo="http://dl.bintray.com/v1/content/uwcirg/truenth"
+
+COPY artifacts /tmp/artifacts
+
+# Only add local repository if packages available in artifacts/
+RUN \
+    if [ -n "$(ls -A /tmp/artifacts)" ]; then \
+        echo deb file:/tmp/artifacts ./ > /etc/apt/sources.list.d/truenth.local.list; \
+    else \
+        echo deb ${debian_repo} stable main > /etc/apt/sources.list.d/truenth.list; \
+    fi
+
+RUN \
+    apt-get update && \
+    apt-get install --yes --force-yes portal && \
+    apt-get clean && \
+    rm --force --recursive --verbose /tmp/artifacts
+
+COPY portal.wsgi /var/www/
+
+ENV PROJECT_DIR="/opt/venvs/portal"
+ENV \
+    FLASK_APP="${PROJECT_DIR}/bin/manage.py" \
+    LOG_FOLDER="/tmp/shared_service_log" \
+    PERSISTENCE_FILE="${PERSISTENCE_FILE:-https://raw.githubusercontent.com/uwcirg/TrueNTH-USA-site-config/master/site_persistence_file.json}" \
+    SQLALCHEMY_DATABASE_URI="postgresql://postgres:@db/postgres" \
+    CELERY_BROKER_URL="redis://redis:6379/0" \
+    REQUEST_CACHE_URL="redis://redis:6379/0" \
+    SESSION_REDIS_URL="redis://redis:6379/0"
+RUN mkdir -p ${LOG_FOLDER} ${PROJECT_DIR}/var/portal-instance/
+RUN chown www-data:www-data ${LOG_FOLDER} ${PROJECT_DIR}/var/portal-instance/
+
+ENV APACHE_SERVER_NAME=${APACHE_SERVER_NAME:-localhost}
+
+RUN a2ensite portal
+
+EXPOSE 80
+
+
+
+CMD \
+    . /etc/apache2/envvars && \
+    . ${PROJECT_DIR}/bin/activate && \
+    if [ -n "$(flask db current)" ]; then \
+        flask db migrate; \
+    else \
+        python ${FLASK_APP} initdb; \
+    fi && \
+    /usr/sbin/apache2 -D FOREGROUND