define and use pinned requirements (#60)

* define and use pinned requirements

* Works with python 3.12

* Build jammy and noble debian packages

.docker/Dockerfile.debian.builder

@@ -1,6 +1,6 @@
 ARG DISTRO=ubuntu:focal
 
-FROM ${DISTRO} as base
+FROM ${DISTRO} AS base
 
 
 RUN apt-get update -qq -o Acquire::Languages=none && \
@@ -28,7 +28,7 @@ COPY . ./
 WORKDIR /dpkg-build
 
 RUN sed -i -re "1s/..UNRELEASED/.ubuntu$(lsb_release -rs)) $(lsb_release -cs)/" debian/changelog \
-    && chmod a-x debian/convertit.* \
-    && dpkg-buildpackage -us -uc -b && mkdir -p /dpkg && cp -pl /convertit[-_]* /dpkg \
+    && chmod a-x debian/convertit.*
+RUN dpkg-buildpackage -us -uc -b && mkdir -p /dpkg && cp -pl /convertit[-_]* /dpkg \
     && dpkg-deb -I /dpkg/convertit*.deb
 WORKDIR /dpkg

.docker/run.sh

@@ -4,6 +4,6 @@ INIPASTE=${INIPASTE:-production.ini}
 
 cd $APP_ROOT
 
-. /opt/apps/convertit/bin/activate
+. /opt/venv/bin/activate
 
-gunicorn --workers=1 --paste=$INIPASTE --bind=0.0.0.0:6543
+pserve $INIPASTE

.github/dependabot.yml

@@ -0,0 +1,12 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 20

.github/workflows/ci.yml

@@ -30,7 +30,7 @@ jobs:
     continue-on-error: true
     name: Check flake8 rules
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: '3.8'
@@ -43,7 +43,7 @@ jobs:
     continue-on-error: true
     name: Check isort rules
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: '3.8'
@@ -56,17 +56,20 @@ jobs:
     continue-on-error: true
     strategy:
       matrix:
-        version: [22.04, 20.04, docker]
+        version: ['24.04', '22.04', '20.04', docker]
         include:
-          - version: 22.04
+          - version: '24.04'
+            distro: noble
+          - version: '22.04'
             distro: jammy
-          - version: 20.04
+          - version: '20.04'
             distro: focal
           - version: docker
             distro: docker image
+
     name: Build ${{ matrix.distro }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Prepare debian versioning
         if: ${{ matrix.version != 'docker' }}
         run: |
@@ -81,7 +84,7 @@ jobs:
 
       - name: Archive package artifact
         if: ${{ matrix.version != 'docker' }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: debian-${{ matrix.distro }}
           path: |
@@ -99,37 +102,40 @@ jobs:
 
       - name: Upload image
         if: ${{ matrix.version == 'docker' }}
-        uses: ishworkh/docker-image-artifact-upload@v1
+        uses: ishworkh/container-image-artifact-upload@v2.0.0
         with:
           image: "convertit:latest"
 
   e2e:
-    continue-on-error: true
     strategy:
       matrix:
-        distro: [ jammy, focal, docker ]
+        distro: [ noble, jammy, focal, docker ]
         include:
+          - distro: noble
+            version: '24.04'
+
           - distro: jammy
-            version: 22.04
+            version: '22.04'
 
           - distro: focal
-            version: 20.04
+            version: '20.04'
 
           - distro: docker
             version: latest
 
     name: Test ${{ matrix.distro }}
     runs-on: ubuntu-${{ matrix.version }}
     needs: [ build ]
+    continue-on-error: true
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         if: ${{ matrix.distro != 'docker' }}
         with:
           name: debian-${{ matrix.distro }}
 
       - name: Download image
         if: ${{ matrix.distro == 'docker' }}
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/container-image-artifact-download@v2.0.0
         with:
           image: "convertit:latest"
 
@@ -146,7 +152,7 @@ jobs:
 
       - name: Test ODT / PDF conversion
         run: |
-          wget -O myfile.pdf "http://localhost:6543/?url=https://file-examples.com/storage/febf69dcf3656dfd992b0fa/2017/10/file-sample_100kB.odt&from=application/vnd.oasis.opendocument.text"
+          wget -O myfile.pdf "http://localhost:6543/?url=https://file-examples.com/wp-content/storage/2017/10/file-sample_100kB.odt&from=application/vnd.oasis.opendocument.text"
           
           if file -b --mime-type ./myfile.pdf |grep -q 'application/pdf'; then
             echo "File is PDF";
@@ -159,7 +165,7 @@ jobs:
 
       - name: Test ODT / DOC conversion
         run: |
-          wget -O myfile.doc "http://localhost:6543/?url=https://file-examples.com/storage/febf69dcf3656dfd992b0fa/2017/10/file-sample_100kB.odt&from=application/vnd.oasis.opendocument.text&to=application/msword"
+          wget -O myfile.doc "http://localhost:6543/?url=https://file-examples.com/storage/fef4e75e176737761a179bf/2017/10/file-sample_100kB.odt&from=application/vnd.oasis.opendocument.text&to=application/msword"
           
           if file -b --mime-type ./myfile.doc |grep -q 'application/msword'; then
             echo "File is DOC";
@@ -195,25 +201,32 @@ jobs:
             file -b -f ./myfile.pdf;
             exit 1;
           fi
+
   deploy:
     name: Publish (on release only)
     runs-on: ubuntu-latest
     needs: [ flake8, isort, e2e ]
     if: ${{ github.event_name == 'release' && github.event.action == 'created' }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+
+      - name: Download 24.04 debian artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: debian-noble
+
       - name: Download 22.04 debian artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: debian-jammy
 
       - name: Download 20.04 debian artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: debian-focal
 
       - name: Download docker image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/container-image-artifact-download@v2.0.0
         with:
           image: "convertit:latest"
 

Dockerfile

@@ -1,29 +1,42 @@
 ARG DISTRO=jammy
 
-FROM ubuntu:${DISTRO} as base
-MAINTAINER Makina Corpus "[email protected]"
+FROM ubuntu:${DISTRO} AS base
+LABEL org.opencontainers.image.authors="Makina Corpus <[email protected]>"
 
 RUN apt-get update && apt-get install -y -qq python3 libreoffice default-jre libreoffice-java-common inkscape python3-magic && \
     apt-get autoclean && apt-get clean all && rm -rf /var/apt/lists/*
 
 
 WORKDIR /opt/apps/convertit
 
-ADD .docker/run.sh /usr/local/bin/run
+COPY .docker/run.sh /usr/local/bin/run
 
 EXPOSE 6543
 CMD ["/bin/sh", "-e", "/usr/local/bin/run"]
 
-FROM base as build
+FROM base AS build
 RUN apt-get update && apt-get install -y -qq build-essential python3-venv python3-dev && \
     apt-get autoclean && apt-get clean all && rm -rf /var/apt/lists/*
 
-ADD . /opt/apps/convertit
+COPY requirements.txt /requirements.txt
 
-RUN python3 -m venv . && ./bin/pip install --no-cache-dir pip setuptools wheel -U
-RUN ./bin/python setup.py install
+RUN python3 -m venv /opt/venv && /opt/venv/bin/pip install --no-cache-dir pip setuptools wheel -U && /opt/venv/bin/pip install --no-cache-dir -U -r /requirements.txt
 
-FROM base as prod
+
+FROM build AS dev
+
+COPY dev-requirements.txt /dev-requirements.txt
+
+RUN /opt/venv/bin/pip install --no-cache-dir -U -r /dev-requirements.txt
+
+FROM base AS prod
 
 COPY --from=build /opt/apps/convertit /opt/apps/convertit
-ADD .docker/run.sh /usr/local/bin/run
+COPY --from=build /opt/venv /opt/venv
+COPY convertit /opt/apps/convertit/convertit
+COPY setup.py /opt/apps/convertit/setup.py
+COPY README.rst /opt/apps/convertit/README.rst
+RUN /opt/venv/bin/pip install .
+COPY production.ini /opt/apps/convertit/production.ini
+VOLUME /var/cache/convertit/downloads
+VOLUME /var/cache/convertit/converted

convertit/converters/unoconv.py

@@ -3,7 +3,6 @@
 import subprocess
 import tempfile
 from functools import partial
-from mimetypes import types_map
 
 from convertit import exists
 
@@ -68,9 +67,9 @@ def is_available():
 def converters():
 
     return {
-        (types_map['.csv'], types_map['.ods']): to_ods,
-        (types_map['.csv'], types_map['.xls']): to_xls,
-        (types_map['.ods'], types_map['.xls']): to_xls,
-        (types_map['.odt'], types_map['.doc']): to_doc,
-        (types_map['.odt'], types_map['.pdf']): to_pdf,
+        ('plain/text', 'application/vnd.oasis.opendocument.spreadsheet'): to_ods,
+        ('plain/text', 'application/vnd.ms-excel'): to_xls,
+        ('application/vnd.oasis.opendocument.spreadsheet', 'application/vnd.ms-excel'): to_xls,
+        ('application/vnd.oasis.opendocument.text', 'application/msword'): to_doc,
+        ('application/vnd.oasis.opendocument.text', 'application/pdf'): to_pdf,
     }

debian/control

@@ -16,6 +16,7 @@ Package: convertit
 Architecture: any
 Depends:
  ${misc:Depends},
+ ${shlibs:Depends},
  python3,
  python3-magic,
  inkscape,

debian/postinst

@@ -2,6 +2,6 @@
 
 adduser --system --group --home /run/convertit --no-create-home --quiet convertit || true
 mkdir -p /var/cache/convertit || true
-chown convertit.convertit /var/cache/convertit || true
+chown convertit:convertit /var/cache/convertit || true
 
 #DEBHELPER#

debian/rules

@@ -9,7 +9,11 @@ override_dh_virtualenv:
 	dh_virtualenv \
 	--python /usr/bin/python3 \
 	--extra-pip-arg --no-cache-dir \
+	--upgrade-pip \
+	--preinstall wheel \
+	--preinstall setuptools \
 	--builtin-venv
+
 	mkdir -p debian/convertit/usr/bin
 	mkdir -p debian/convertit/opt/convertit
 	cp production.ini debian/convertit/opt/convertit/convertit.ini

dev-requirements.in

@@ -0,0 +1,9 @@
+-c requirements.txt
+
+webtest
+mock
+coverage
+freezegun
+flake8
+isort
+pip-tools

dev-requirements.txt

@@ -0,0 +1,56 @@
+#
+# This file is autogenerated by pip-compile with Python 3.12
+# by the following command:
+#
+#    pip-compile dev-requirements.in
+#
+beautifulsoup4==4.12.2
+    # via webtest
+build==1.0.3
+    # via pip-tools
+click==8.1.7
+    # via pip-tools
+coverage==7.3.2
+    # via -r dev-requirements.in
+flake8==6.1.0
+    # via -r dev-requirements.in
+freezegun==1.3.1
+    # via -r dev-requirements.in
+isort==5.12.0
+    # via -r dev-requirements.in
+mccabe==0.7.0
+    # via flake8
+mock==5.1.0
+    # via -r dev-requirements.in
+packaging==23.2
+    # via
+    #   -c requirements.txt
+    #   build
+pip-tools==7.3.0
+    # via -r dev-requirements.in
+pycodestyle==2.11.1
+    # via flake8
+pyflakes==3.1.0
+    # via flake8
+pyproject-hooks==1.0.0
+    # via build
+python-dateutil==2.8.2
+    # via freezegun
+six==1.16.0
+    # via python-dateutil
+soupsieve==2.5
+    # via beautifulsoup4
+waitress==2.1.2
+    # via webtest
+webob==1.8.7
+    # via
+    #   -c requirements.txt
+    #   webtest
+webtest==3.0.0
+    # via -r dev-requirements.in
+wheel==0.42.0
+    # via pip-tools
+
+# The following packages are considered to be unsafe in a requirements file:
+# pip
+# setuptools

docker-compose.yml

@@ -0,0 +1,7 @@
+version: "3.8"
+services:
+  web:
+    build:
+      target: dev
+    volumes:
+      - .:/opt/apps/convertit

production.ini

@@ -35,7 +35,7 @@ convertit.converters =
 
 [server:main]
 use = egg:gunicorn#main
-host = localhost
+host = 0.0.0.0
 port = 6543
 
 # Begin logging configuration

requirements.in

@@ -0,0 +1,4 @@
+pyramid
+python-magic
+Pillow
+gunicorn