From 9114cb4e1b376e806c5aaceed1ff1cb4fc024cb0 Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Thu, 20 May 2021 01:06:20 +0530 Subject: [PATCH 1/2] Eliminate postgresql docker container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Initializing docker containers used to take 23-25s alone. α Now Initializing postgresql (Setup database) takes 4-6s. β α: https://github.com/nexB/vulnerablecode/runs/2623306124?check_suite_focus=true https://github.com/nexB/vulnerablecode/runs/2623290485?check_suite_focus=true https://github.com/nexB/vulnerablecode/runs/2594778880?check_suite_focus=true β: https://github.com/Hritik14/vulnerablecode/runs/2631513782?check_suite_focus=true https://github.com/Hritik14/vulnerablecode/runs/2631540610?check_suite_focus=true https://github.com/Hritik14/vulnerablecode/runs/2631552958?check_suite_focus=true Signed-off-by: Hritik Vijay --- .github/workflows/main.yml | 40 ++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d05b5f0ba..321fe875b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,22 +6,6 @@ jobs: unit_tests: runs-on: ubuntu-latest - services: - # Label used to access the service container - postgres: - image: postgres - env: - POSTGRES_PASSWORD: vulnerablecode - POSTGRES_DB: vulnerablecode - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - # Maps tcp port 5432 on service container to the host - - 5432:5432 steps: - name: Check out repository code uses: actions/checkout@v2 @@ -31,18 +15,36 @@ jobs: with: python-version: 3.8 + # - name: Restore cache + # uses: actions/cache@v2 + # with: + # path: ~/.cache/pip + # key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + # restore-keys: | + # ${{ runner.os }}-pip- + - name: Install dependencies run: | - sudo apt install python3-dev postgresql libpq-dev build-essential libxml2-dev libxslt1-dev + sudo apt install python3-dev postgresql libpq-dev build-essential libxml2-dev libxslt1-dev postgresql ncat python -m pip install --upgrade pip pip install -r requirements.txt + - name: Setup database + env: + PGPASSWORD: vulnerablecode + run: | + sudo systemctl start postgresql + sudo -Eu postgres psql -c "CREATE ROLE vulnerablecode WITH PASSWORD '$PGPASSWORD' NOSUPERUSER CREATEDB NOCREATEROLE INHERIT LOGIN;" + sudo systemctl status postgresql + createdb --encoding=utf-8 --owner=vulnerablecode --user=vulnerablecode \ + --host=localhost --port=5432 vulnerablecode + - name: Run tests run: python -m pytest -v -m "not webtest" env: - # The hostname, username used to communicate with the PostgreSQL service container + # The hostname, username used to communicate with Postgresql POSTGRES_HOST: localhost - VC_DB_USER: postgres + VC_DB_USER: vulnerablecode POSTGRES_PORT: 5432 DJANGO_DEV: 1 GH_TOKEN: 1 From 135c95f7bd7eb7bccede20366b6cda750d6eb213 Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Thu, 20 May 2021 22:25:42 +0530 Subject: [PATCH 2/2] Use cache for pip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Earlier Install Dependencies used to take 50-60s α Now it takes ~10s including cache retrival time β The total workflow time has reduced from ~1:40 to ~0:55 α: https://github.com/nexB/vulnerablecode/runs/2623306124?check_suite_focus=true https://github.com/nexB/vulnerablecode/runs/2623290485?check_suite_focus=true https://github.com/nexB/vulnerablecode/runs/2594778880?check_suite_focus=true β: https://github.com/Hritik14/vulnerablecode/runs/2632140380?check_suite_focus=true https://github.com/Hritik14/vulnerablecode/runs/2632128817?check_suite_focus=true https://github.com/Hritik14/vulnerablecode/runs/2632109016?check_suite_focus=true Signed-off-by: Hritik Vijay --- .github/workflows/main.yml | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 321fe875b..9966ac59f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,19 +15,29 @@ jobs: with: python-version: 3.8 - # - name: Restore cache - # uses: actions/cache@v2 - # with: - # path: ~/.cache/pip - # key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - # restore-keys: | - # ${{ runner.os }}-pip- + - name: Restore cache + uses: actions/cache@v2 + with: + path: .venv + key: ${{ runner.os }}-venv-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-venv- - name: Install dependencies run: | sudo apt install python3-dev postgresql libpq-dev build-essential libxml2-dev libxslt1-dev postgresql ncat python -m pip install --upgrade pip - pip install -r requirements.txt + + - uses: syphar/restore-virtualenv@v1 + id: cache-virtualenv + with: + requirement_files: requirements.txt # this is optional + + - uses: syphar/restore-pip-download-cache@v1 + if: steps.cache-virtualenv.outputs.cache-hit != 'true' + + - run: pip install -r requirements.txt + if: steps.cache-virtualenv.outputs.cache-hit != 'true' - name: Setup database env: @@ -42,9 +52,5 @@ jobs: - name: Run tests run: python -m pytest -v -m "not webtest" env: - # The hostname, username used to communicate with Postgresql - POSTGRES_HOST: localhost - VC_DB_USER: vulnerablecode - POSTGRES_PORT: 5432 DJANGO_DEV: 1 GH_TOKEN: 1