Changes to get the GenomicsDB Python bindings packaged and published (#…

…32)

package/Dockerfile

@@ -1,5 +1,10 @@
+#
+# Dockerfile
+#
 # The MIT License (MIT)
+#
 # Copyright (c) 2020 Omics Data Automation, Inc.
+# Copyright (c) 2023 dātma, inc™
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in
@@ -19,8 +24,9 @@
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #
 # Description: Base Docker file for creating PyPi Packages
+#
 
-FROM ghcr.io/genomicsdb/genomicsdb:centos_6_prereqs
+FROM centos:7
 
 ARG user=genomicsdb
 ARG user_id=0
@@ -29,12 +35,13 @@ ARG group_id=0
 ARG genomicsdb_branch=develop
 
 COPY scripts /build
-WORKDIR /build
-RUN ./install_python.sh $user $user_id $group_id
 
 WORKDIR /build
 RUN ./install_genomicsdb.sh $user $genomicsdb_branch
 
+WORKDIR /build
+RUN ./install_python.sh $user $user_id $group_id
+
 USER ${user}
 WORKDIR /home/${user}
 ENTRYPOINT ["/bin/bash", "--login"]

package/docker-compose.yml

@@ -1,8 +1,34 @@
+#
+# docker-compose.yml
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2023 dātma, inc™
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+# Description: Base Docker file for creating PyPi Packages
+
 version: "3"
 services:
   package:
     image: genomicsdb:python
-    user: ${CURRENT_UID}
+    user: ${USER}
     volumes: 
       - ..:/home/${USER}/GenomicsDB-Python:delegated
     command: "/home/${USER}/GenomicsDB-Python/package/scripts/create_package.sh"

package/publish_package.sh

@@ -1,3 +1,30 @@
+#
+# publish_package.sh
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2023 dātma, inc™
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+# Description: Bash script to help with packaging and releasing to PyPi
+#
+
 #!/bin/bash
 
 USAGE="Usage: publish_package.sh release|test-release macos|linux" 
@@ -43,9 +70,13 @@ docker create -it --name genomicsdb genomicsdb:python bash &&
   mv genomicsdb/protobuf/python/* genomicsdb/protobuf &&
   rmdir genomicsdb/protobuf/python &&
   mkdir -p genomicsdb/lib &&
-  docker cp genomicsdb:/usr/local/lib/libtiledbgenomicsdb.so genomicsdb/lib &&
+  docker cp -L genomicsdb:/usr/local/lib/libtiledbgenomicsdb.so genomicsdb/lib &&
   docker rm -fv genomicsdb &&
-  sed -i 's/import genomicsdb_/from . import genomicsdb_/g' genomicsdb/protobuf/*.py &&
+  if [[ $(uname) == "Darwin" ]]; then
+    sed -i '' 's/import genomicsdb_/from . import genomicsdb_/g' genomicsdb/protobuf/*.py
+  else
+    sed -i 's/import genomicsdb_/from . import genomicsdb_/g' genomicsdb/protobuf/*.py
+  fi &&
   echo "Docker copy from genomicsdb:python successful"
 
 RC=$?
@@ -59,19 +90,19 @@ python setup.py sdist
 
 popd
 
-# Use centos6 based genomicsdb:all_python Docker image to create packages for 3.9/3.10/1.11
-echo "Building packages for Linux on CentOS 6..."
-export CURRENT_UID="$(id -u):$(id -g)"
+# Use centos6 based genomicsdb:all_python Docker image to create packages for 3.9/3.10/3.11
+echo "Building packages for Linux on CentOS 7..."
 docker-compose run -e PYTHON_VERSION="3.9" package
 docker-compose run -e PYTHON_VERSION="3.10" package
 docker-compose run -e PYTHON_VERSION="3.11" package
-echo "Building packages for Linux on CentOS 6 DONE"
+echo "Building packages for Linux on CentOS 7 DONE"
 
 pushd ../
 
 # Repair linux wheel names
 for linux_wheel in dist/*-linux_*.whl; do
-  mv $linux_wheel ${linux_wheel//-linux_/-manylinux1_};
+  echo "Moving $linux_wheel to ${linux_wheel//-linux_/-manylinux_2_17_x86_64.manylinux2014_}"
+  mv $linux_wheel ${linux_wheel//-linux_/-manylinux_2_17_x86_64.manylinux2014_}
 done
 
 # Publish

package/publish_package_local.sh

@@ -1,13 +1,40 @@
+#
+# publish_package_local.sh
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2023 dātma, inc™
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+# Description: Bash script to publish locally on MacOS
+#
+
 #!/bin/bash
 
-PREFIX_DIR=$HOME/python
+PREFIX_DIR=${PREFIX_DIR:-$(mktemp -d)}
 GENOMICSDB_DIR=$PREFIX_DIR/GenomicsDB
 GENOMICSDB_HOME=$GENOMICSDB_DIR/release
-GENOMICSDB_BRANCH=master
+GENOMICSDB_BRANCH=${GENOMICSDB_BRANCH:-master}
 
 GENOMICSDB_PYTHON_DIR=$(readlink -f `pwd`/..)
 
-#MACOSX_DEPLOYMENT_TARGET=11.0
+MACOSX_DEPLOYMENT_TARGET=12.1
 HOMEBREW_NO_AUTO_UPDATE=1
 
 die() {
@@ -25,9 +52,15 @@ check_rc() {
   fi
 }
 
-install_openssl() {
-  check_rc $(brew list [email protected] &> /dev/null || brew install [email protected])
-  OPENSSL_ROOT_DIR=$(readlink -f /usr/local/opt/[email protected])
+install_prereqs() {
+  HOMEBREW_NO_AUTO_UPDATE=1
+  HOMEBREW_NO_INSTALL_CLEANUP=1
+  check_rc $(brew list openssl@3 &> /dev/null || brew install openssl@3)
+  OPENSSL_ROOT_DIR=$(brew --prefix openssl@3)
+  brew list libcsv &> /dev/null && brew uninstall libcsv
+  # Use the uuid from framework
+  brew list ossp-uuid &> /dev/null && brew uninstall ossp-uuid
+
 }
 
 install_python_version() {
@@ -43,11 +76,9 @@ publish_package() {
 		source try$PYTHON_VERSION/bin/activate &&
 		pip install --upgrade pip &&
     pushd $GENOMICSDB_PYTHON_DIR &&
-	  pip install -r requirements.txt &&
-    cp $GENOMICSDB_HOME/genomicsdb/protobuf/python/* genomicsdb/protobuf &&
-    sed -i '' 's/import genomicsdb_/from . import genomicsdb_/g' genomicsdb/protobuf/*.py &&  
+    pip install -r package/requirements_pkg.txt &&
     python setup.py sdist --with-genomicsdb=$GENOMICSDB_HOME &&
-	  python setup.py bdist_wheel --with-genomicsdb=$GENOMICSDB_HOME --with-libs &&
+	  python setup.py bdist_wheel --with-genomicsdb=$GENOMICSDB_HOME --with-libs --with-protobuf &&
     popd &&
 		deactivate
 	RC=$?
@@ -59,26 +90,36 @@ publish_package() {
 
 publish() {
   echo "Installing Python" &&
-#  publish_package 3.7 &&
-#	publish_package 3.8 &&
-  publish_package 3.9
+  publish_package 3.9 &&
+  publish_package 3.10 &&
+  publish_package 3.11
 }
 
 install_genomicsdb() {
+  echo "Building GenomicsDB in $GENOMICSDB_DIR..."
   rm -fr $GENOMICSDB_DIR
   git clone https://github.com/GenomicsDB/GenomicsDB.git -b $GENOMICSDB_BRANCH $GENOMICSDB_DIR
   pushd $GENOMICSDB_DIR
   mkdir build
   cd build
-  cmake -DCMAKE_INSTALL_PREFIX=$GENOMICSDB_HOME -DCMAKE_PREFIX_PATH=$OPENSSL_ROOT_DIR -DBUILD_EXAMPLES=False -DDISABLE_MPI=True -DBUILD_DISTRIBUTABLE_LIBRARY=1 -DBUILD_FOR_PYTHON=1 .. &&
-    make -j4 && make install
+  cmake -DCMAKE_INSTALL_PREFIX=$GENOMICSDB_HOME -DCMAKE_PREFIX_PATH=$OPENSSL_ROOT_DIR -DPROTOBUF_ROOT_DIR=./protobuf-install -DAWSSDK_ROOT_DIR=./aws-install -DGCSSDK_ROOT_DIR=./gcs-install -DBUILD_EXAMPLES=False -DDISABLE_MPI=True -DBUILD_DISTRIBUTABLE_LIBRARY=1 -DBUILD_FOR_PYTHON=1 .. &&
+    make && make install
+  popd
+  if [[ -f $GENOMICSDB_HOME/lib/libtiledbgenomicsdb.dylib ]]; then
+    echo "Building GenomicsDB in $GENOMICSDB_DIR DONE"
+  else
+    echo "Something wrong with building GenomicsDB ar $GENOMICSDB_HOME"
+    exit 1
+  fi
 }
 
 if [[ `uname` != "Darwin" ]]; then
   echo "Script needs porting for Non-MacOS systems"
   exit 1
 fi
 
-install_openssl &&
-  install_genomicsdb &&
+install_prereqs
+install_genomicsdb &&
   publish
+
+rm -fr $GENOMICSDB_DIR

package/requirements_pkg.txt

@@ -1 +1,39 @@
--r ../requirements_pkg.txt
+#
+# requirements_pkg.txt
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2023 dātma, inc™
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+# Description: Dependencies for packaging up the bindings
+#
+
+-r ../requirements.txt
+
+# C/C++ bindings
+cython>=0.29.31
+
+# Building
+setuptools>=18.0.1
+setuptools-scm>=1.5.4
+wheel>=0.37.0
+
+# Publishing
+twine

package/scripts/create_package.sh

@@ -1,3 +1,31 @@
+#
+# create_package.sh
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2023 dātma, inc™
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+# Description: Bash script to be used with docker compose for creating a python
+#              package ready for publishing
+#
+
 #!/bin/bash
 
 set -e
@@ -13,33 +41,26 @@ if [ ! -d GenomicsDB-Python ]; then
   exit 1
 fi
 
-if [ -d /usr/local/ssl/lib ]; then
-	export LD_LIBRARY_PATH="/usr/local/ssl/lib:$LD_LIBRARY_PATH"
-else
-	echo "/usr/local/ssl/lib not found. Cannot continue."
-	exit 1
-fi
+source /opt/rh/devtoolset-11/enable
+export LD_LIBRARY_PATH="/usr/local/lib:/usr/local/lib64::$LD_LIBRARY_PATH"
 
+cd GenomicsDB-Python
+echo "PWD=$(pwd)"
 echo "Packaging genomicsdb for Python Version=$PYTHON_VERSION..."
 
 python$PYTHON_VERSION -m venv env-dist-$PYTHON_VERSION &&
   source env-dist-$PYTHON_VERSION/bin/activate &&
   echo "Installing required dependencies for $PYTHON_VERSION..." &&
   pip install --upgrade pip &&
-  echo "Installing wheel..." && pip install wheel && echo "Installing wheel DONE" &&
-  echo "Installing setuptools..." && pip install setuptools && echo "Installing setuptools DONE" &&
-  echo "Installing setuptools-scm..." && pip install setuptools-scm && echo "Installing setuptools-scm DONE" &&
-  echo "Installing numpy..." && pip install numpy && echo "Installing numpy DONE" &&
-  echo "Installing cython..." && pip install cython && echo "Installing cython DONE" &&
+  pip install -r package/requirements_pkg.txt &&
   echo "Installing required dependencies for $PYTHON_VERSION DONE"
 
 if [[ $? -ne 0 ]]; then
   echo "Problems installing dependencies. Cannot continue"
   exit 1
 fi
 
-cd GenomicsDB-Python &&
-  python setup.py bdist_wheel --python-tag=cp${PYTHON_VERSION//./} &&
+python setup.py bdist_wheel --python-tag=cp${PYTHON_VERSION//./} &&
   echo "Packaging genomicsdb for Python Version=$PYTHON_VERSION DONE" &&
   exit 0