diff --git a/.github/workflows/test-import-using-nix.yml b/.github/workflows/test-import-using-nix.yml index eca46687c..b2477f2a5 100644 --- a/.github/workflows/test-import-using-nix.yml +++ b/.github/workflows/test-import-using-nix.yml @@ -27,4 +27,4 @@ jobs: cd etc/nix ./get-latest-pypi-deps-db.sh --in-place nix --print-build-logs flake check - ./test-import-using-nix.sh alpine + nix-shell --run ./test-import-using-nix.sh diff --git a/etc/nix/flake.lock b/etc/nix/flake.lock index 89db951b6..cb3694fa3 100644 --- a/etc/nix/flake.lock +++ b/etc/nix/flake.lock @@ -22,34 +22,33 @@ "pypi-deps-db": "pypi-deps-db" }, "locked": { - "lastModified": 1614316305, - "narHash": "sha256-GLZqAazQu7zYR/1VU8QpjdrPdNMn9H8QdlMukT1TeZQ=", + "lastModified": 1643953409, + "narHash": "sha256-CJDg/RpZdUVyI3QIAXUqIoYDl7VkxFtNE4JWih0ucKc=", "owner": "DavHau", "repo": "mach-nix", - "rev": "235a0a81d05a043bca2a93442f2560946266fc73", + "rev": "fe5255e6fd8df57e9507b7af82fc59dda9e9ff2b", "type": "github" }, "original": { "owner": "DavHau", - "ref": "235a0a81d05a043bca2a93442f2560946266fc73", + "ref": "fe5255e6fd8df57e9507b7af82fc59dda9e9ff2b", "repo": "mach-nix", "type": "github" } }, "nixpkgs": { "locked": { - "lastModified": 1605988311, - "narHash": "sha256-PA+kgq46NApOAJlmBNJHs5DwsIrY+jodM0e4g7VtXyY=", + "lastModified": 1645334861, + "narHash": "sha256-We9ECiMglthzbZ5S6Myqqf+RHzBFZPoM2qL5/jDkUjs=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "2247d824fe07f16325596acc7faa286502faffd1", + "rev": "d5f237872975e6fb6f76eef1368b5634ffcd266f", "type": "github" }, "original": { - "owner": "NixOS", + "id": "nixpkgs", "ref": "nixos-unstable", - "repo": "nixpkgs", - "type": "github" + "type": "indirect" } }, "nixpkgs_2": { diff --git a/etc/nix/flake.nix b/etc/nix/flake.nix index f408e7e98..b9fd527d6 100644 --- a/etc/nix/flake.nix +++ b/etc/nix/flake.nix @@ -13,7 +13,7 @@ type = "github"; owner = "DavHau"; repo = "mach-nix"; - ref = "235a0a81d05a043bca2a93442f2560946266fc73"; + ref = "fe5255e6fd8df57e9507b7af82fc59dda9e9ff2b"; # 3.4.0 }; outputs = { self, nixpkgs, machnix }: @@ -29,9 +29,6 @@ version = builtins.head (builtins.match ''.*version=["']?([^"',]+).*'' (builtins.readFile (vulnerablecode-src + "/setup.py"))); - # Common shell code. - libSh = ./lib.sh; - # System types to support. supportedSystems = [ "x86_64-linux" ]; @@ -57,9 +54,10 @@ # mach-nix release) is usually insufficient. Use # ./get-latest-pypi-deps-db.sh to obtain the data rev & hash. pypiDataRev = - "8dcec158c51f8a96f316630679222e436c1b078c"; # 2021-06-16T08:41:20Z + "897a7471aa4e83aab21d2c501e00fee3f440e0fe"; # 2022-02-21T08:57:22Z pypiDataSha256 = - "0499zl39aia74f0i7fkn5dsy8244dkmcw4vzd5nf4kai605j2jli"; + "03gnaq687gg9afb6i6czw4kzr1gbnzna15lfb26f9nszyfq3iyaj"; + }); # This wrapper allows to setup both the production as well as the # development Python environments in the same way (albeit having @@ -69,11 +67,6 @@ requirements = '' ${requirements} ''; - # Fix an issue with an upstream dep of GitPython. - # https://github.com/DavHau/mach-nix/issues/287 - # See https://github.com/DavHau/mach-nix/issues/318 - _.gitpython.propagatedBuildInputs.mod = pySelf: self: oldVal: - oldVal ++ [ pySelf.typing-extensions ]; }; in { @@ -91,12 +84,6 @@ dontBuild = true; # do not use Makefile propagatedBuildInputs = [ pythonEnv postgresql gitMinimal ]; - postPatch = '' - # Do not use absolute path. - substituteInPlace vulnerablecode/settings.py \ - --replace 'STATIC_ROOT = "/var/vulnerablecode/static"' 'STATIC_ROOT = "./static"' - ''; - installPhase = '' cp -r . $out ''; @@ -144,25 +131,24 @@ unpackPhase = "true"; buildPhase = '' - source ${libSh} - initPostgres $(pwd) - export SECRET_KEY=REALLY_SECRET - ${vulnerablecode}/manage.py collectstatic --no-input - ${vulnerablecode}/manage.py migrate + # Work on a local copy. + cp -r ${vulnerablecode} ./vulnerablecode + cd ./vulnerablecode + chmod -R +w . + + source ./etc/nix/lib.sh + + setupDevEnv ''; doCheck = true; checkPhase = '' - # Run pytest on the installed version. A running postgres - # database server is needed. - ( - cd ${vulnerablecode} - black -l 100 --check . - pytest -m "not webtest" - ) + export PYTHON_EXE=${pythonEnvDev}/bin/python3 # use correct python + make check + make test # Launch the webserver and call the API. - ${vulnerablecode}/manage.py runserver & + make run & sleep 2 wget http://127.0.0.1:8000/api/ kill %1 # kill background task (i.e. webserver) diff --git a/etc/nix/lib.sh b/etc/nix/lib.sh index fc75cf5c9..127b28998 100644 --- a/etc/nix/lib.sh +++ b/etc/nix/lib.sh @@ -1,18 +1,18 @@ #!/usr/bin/env bash -# Setup postgres; see the README for the latest instructions. -# -# $RUNDIR is used to prevent postgres from accessings its default run dir at -# /run/postgresql. See -# https://github.com/NixOS/nixpkgs/issues/83770#issuecomment-607992517 -function initPostgres() { - ROOTDIR=$1 - DATADIR=$ROOTDIR/pgdata - RUNDIR=$ROOTDIR/run - ENCODING="UTF-8" - mkdir -p "$RUNDIR" - initdb -D "$DATADIR" -E $ENCODING - pg_ctl -D "$DATADIR" -o "-k $RUNDIR" -l "$DATADIR/logfile" start - createuser --host "$RUNDIR" --no-createrole --no-superuser --login --inherit --createdb vulnerablecode - createdb --host "$RUNDIR" -E $ENCODING --owner=vulnerablecode --user=vulnerablecode --port=5432 vulnerablecode +# Setup dev environment; see the README for the latest instructions. +setupDevEnv() { + # Make sure postgres uses a local socket file. The posgres + # commands (initd,b createdb, createuser, etc.) honor these + # settings. + export PGHOST=$PWD + export PGDATA=./pgdata + # Start postgres. + initdb -E utf-8 + pg_ctl -o "-k $PGHOST" -l ./logfile start + + # Setup dev environment. + export ACTIVATE= # no venv + sed -i 's/sudo -u postgres//' Makefile # no extra user + make envfile postgres } diff --git a/etc/nix/test-import-using-nix.sh b/etc/nix/test-import-using-nix.sh index f66c375a2..9613e83f7 100755 --- a/etc/nix/test-import-using-nix.sh +++ b/etc/nix/test-import-using-nix.sh @@ -4,27 +4,29 @@ # Populate a test database using either the Nix installation or the local # checkout. -set -e +set -exv THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" DEFAULT_INSTALL_DIR=$VULNERABLECODE_INSTALL_DIR # in the Nix store, see flake.nix INSTALL_DIR=${INSTALL_DIR:-$DEFAULT_INSTALL_DIR} ARGS=$(if [ $# -eq 0 ]; then echo "--all"; else echo "$@"; fi) -export SECRET_KEY=REALLY_SECRET -TEMPDIR=$(mktemp -d -p "$THIS_DIR") -export TEMPDIR source "$THIS_DIR/lib.sh" cleanup() { - pg_ctl -D "$DATADIR" stop + pg_ctl stop rm -rf "$TEMPDIR" } trap cleanup EXIT -initPostgres "$TEMPDIR" - -"$INSTALL_DIR/manage.py" migrate -"$INSTALL_DIR/manage.py" collectstatic --no-input -"$INSTALL_DIR/manage.py" import $ARGS +TEMPDIR=$(mktemp -d -p "$THIS_DIR") +cp -r "$INSTALL_DIR" "$TEMPDIR/vulnerablecode" +cd "$TEMPDIR/vulnerablecode" +chmod -R +w . +setupDevEnv + +./manage.py migrate +./manage.py collectstatic --no-input +./manage.py import $ARGS +./manage.py improve $ARGS