Merge branch 'develop'

CHANGELOG.md

@@ -1,3 +1,9 @@
+### citustools v0.7.10 (May 28, 2019) ###
+
+* Add custom branch support for valgrind tests
+
+* Add `--diff` option to `citus_indent`
+
 ### citustools v0.7.9 (August 15, 2018) ###
 
 * Update PG11 build to use `REL_11_STABLE`, not `master`

uncrustify/citus_indent

@@ -9,15 +9,22 @@ use POSIX qw(setlocale LC_ALL);
 local $ENV{'PATH'} = '/usr/local/bin:/usr/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin';
 delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
 
-my ($quiet, $check);
-exit 64 unless GetOptions ('quiet' => \$quiet, 'check' => \$check);
+my ($quiet, $check, $diff);
+exit 64 unless GetOptions ('quiet' => \$quiet, 'check' => \$check, 'diff' => \$diff);
 
-my $null_delimited_triples = `git ls-files -z | git check-attr --stdin -z citus-style`;
+my $null_delimited_triples = ($diff ? 
+    `git diff --cached --name-only -z | git check-attr --stdin -z citus-style` : 
+                     `git ls-files -z | git check-attr --stdin -z citus-style`
+);
 my @flattened_triples = split(/\x00/, $null_delimited_triples);
 my $exit_code = $? >> 8;
 
 die "could not list files under source control\n" if $exit_code;
 
+# exit if there are no files to check
+print "no files to check" if !@flattened_triples && !$quiet;
+exit if !@flattened_triples;
+
 my @files_to_format = ();
 my @formatter_args = qw(-c /usr/local/etc/citustools/citus-style.cfg);
 
@@ -88,6 +95,11 @@ Do not modify any files, instead simply verify that all files eligible for
 formatting are compliant with Citus Data style. The exit code I<EXIT_SUCCESS>
 signals that a codebase is compliant, otherwise I<EXIT_FAILURE> is used.
 
+=item B<--diff>
+
+Only check files that are staged for commit. This is primarily useful when used
+in a pre-commit hook as it is generally faster to only check the changed files.
+
 =back
 
 =head1 FILES

valgrind/launch-test-instance

@@ -2,6 +2,21 @@
 
 set -euo pipefail
 
+REPORT_EMAIL=${REPORT_EMAIL:-burak@citusdata.com metin@citusdata.com furkan@citusdata.com}
+POSTGRES_GITREF=${POSTGRES_GITREF:-REL_11_STABLE}
+CITUS_GITREF=${CITUS_GITREF:-master}
+
+echo "ENV:"
+echo "     REPORT_EMAIL:" $REPORT_EMAIL
+echo "  POSTGRES_GITREF:" $POSTGRES_GITREF
+echo "     CITUS_GITREF:" $CITUS_GITREF
+
+# when running in a tty wait for the human to press enter, this is to allow the human to verify the settings
+# when running as a crontab it will just continue
+if [ -t 1 ] ; then
+    read -p "Press enter to continue"
+fi
+
 # create a key pair just for valgrind tests and store it in valgrind-test.pem
 echo "Creating key pair..."
 key_name=valgrind_$RANDOM
@@ -16,9 +31,10 @@ valgrind_instance_id=$(aws ec2 run-instances \
     --instance-type r3.2xlarge \
     --key-name $key_name \
     --instance-initiated-shutdown-behavior terminate \
-    --user-data file:///usr/local/bin/download-test-scripts \
+    --user-data file://download-test-scripts \
     --query 'Instances[0].InstanceId' \
     --output text)
+echo "  instance id:" $valgrind_instance_id
 
 # tag the instance as ValgrindTest
 echo "Tagging the instance..."
@@ -41,7 +57,19 @@ valgrind_instance_ip=$(aws ec2 describe-instances \
 # run valgrind tests
 echo "Running the valgrind tests..."
 echo "This will take hours, test results will be sent via e-mail."
-ssh -o StrictHostKeyChecking=no -i $key_name.pem ubuntu@$valgrind_instance_ip "screen -d -m run-valgrind-tests"
+ssh \
+    -o IdentitiesOnly=yes \
+    -o StrictHostKeyChecking=no \
+    -i $key_name.pem \
+    ubuntu@$valgrind_instance_ip \
+    REPORT_EMAIL="${REPORT_EMAIL}" \
+    POSTGRES_GITREF="${POSTGRES_GITREF}" \
+    CITUS_GITREF="${CITUS_GITREF}" \
+    screen -d -m run-valgrind-tests
+
+echo login:
+echo "  " ssh -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -i $key_name.pem ubuntu@$valgrind_instance_ip
+echo
 
 #delete the key pair after we are done with tests
 aws ec2 delete-key-pair --key-name $key_name

valgrind/run-valgrind-tests

@@ -2,6 +2,17 @@
 
 set -euo pipefail
 
+REPORT_EMAIL=${REPORT_EMAIL:-burak@citusdata.com metin@citusdata.com furkan@citusdata.com}
+CITUS_GITREF=${CITUS_GITREF:-master}
+POSTGRES_GITREF=${POSTGRES_GITREF:-REL_11_STABLE}
+
+echo "ENV:"
+echo "     REPORT_EMAIL:" $REPORT_EMAIL
+echo "  POSTGRES_GITREF:" $POSTGRES_GITREF
+echo "     CITUS_GITREF:" $CITUS_GITREF
+
+ulimit -c unlimited
+
 # download and install required packages
 sudo apt-get update
 sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq \
@@ -20,31 +31,30 @@ sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq \
 export LC_ALL=en_US.UTF-8
 export LANG=en_US.UTF-8
 export LANGUAGE=en_US.UTF-8
-export PG_CONFIG=/usr/local/pgsql/bin/pg_config
+export PATH=$HOME/pgsql/bin:$PATH
 
 # download and install PostgreSQL
-git clone -b "REL_10_STABLE" --depth 1 git://git.postgresql.org/git/postgresql.git
+git clone -b "${POSTGRES_GITREF}" --depth 1 git://git.postgresql.org/git/postgresql.git
 cd postgresql/
-./configure --enable-cassert --enable-debug CFLAGS="-ggdb -Og -DUSE_VALGRIND"
+./configure \
+    --prefix=$HOME/pgsql \
+    --with-openssl \
+    --enable-cassert \
+    --enable-debug \
+    CFLAGS="-ggdb -Og -DUSE_VALGRIND"
 
 # we will use this to parallelize PostgreSQL compilation
 procs="$(nproc)"
 mjobs="$((procs + 1))"
-make -j "${mjobs}" -s
-sudo make install
-export PATH=/usr/local/pgsql/bin:$PATH
+make install -j "${mjobs}" -s
 
 # download and install Citus
 cd ..
-git clone https://github.com/citusdata/citus.git
+git clone -b "${CITUS_GITREF}" --depth 1 https://github.com/citusdata/citus.git
 cd citus/
 ./configure
 make clean
-make -j8 -s
-sudo make install
-
-# this is necessary to start tests
-sudo chown ubuntu /usr/local/pgsql/bin/ -R
+make install -j "${mjobs}" -s
 
 # run valgrind tests
 cd src/test/regress
@@ -60,9 +70,9 @@ if [ -s regression.diffs ]; then
 fi
 
 if [ -z "$attachments" ]; then
-    mail -aFrom:[email protected] -s "[Valgrind Test Results] - Success" [email protected] [email protected] [email protected] < /dev/null
+    mail -aFrom:[email protected] -s "[Valgrind Test Results] - Success" $REPORT_EMAIL < /dev/null
 else
-    mail -aFrom:[email protected] -s "[Valgrind Test Results] - Failure" $attachments [email protected] [email protected] [email protected] < /dev/null
+    mail -aFrom:[email protected] -s "[Valgrind Test Results] - Failure" $attachments $REPORT_EMAIL < /dev/null
 fi
 
 # just to ensure everything is completed in the test instance