diff --git a/.docker/Dockerfile.debian.builder b/.docker/Dockerfile.debian.builder index 249500e..644264e 100644 --- a/.docker/Dockerfile.debian.builder +++ b/.docker/Dockerfile.debian.builder @@ -1,6 +1,6 @@ ARG DISTRO=ubuntu:focal -FROM ${DISTRO} as base +FROM ${DISTRO} AS base RUN apt-get update -qq -o Acquire::Languages=none && \ @@ -28,7 +28,7 @@ COPY . ./ WORKDIR /dpkg-build RUN sed -i -re "1s/..UNRELEASED/.ubuntu$(lsb_release -rs)) $(lsb_release -cs)/" debian/changelog \ - && chmod a-x debian/convertit.* \ - && dpkg-buildpackage -us -uc -b && mkdir -p /dpkg && cp -pl /convertit[-_]* /dpkg \ + && chmod a-x debian/convertit.* +RUN dpkg-buildpackage -us -uc -b && mkdir -p /dpkg && cp -pl /convertit[-_]* /dpkg \ && dpkg-deb -I /dpkg/convertit*.deb WORKDIR /dpkg diff --git a/.docker/run.sh b/.docker/run.sh index 6d483f1..bdd782e 100644 --- a/.docker/run.sh +++ b/.docker/run.sh @@ -4,6 +4,6 @@ INIPASTE=${INIPASTE:-production.ini} cd $APP_ROOT -. /opt/apps/convertit/bin/activate +. /opt/venv/bin/activate -gunicorn --workers=1 --paste=$INIPASTE --bind=0.0.0.0:6543 +pserve $INIPASTE diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..1b18eec --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" # Location of package manifests + schedule: + interval: "weekly" + open-pull-requests-limit: 20 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dabaea6..08e21e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: continue-on-error: true name: Check flake8 rules steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: '3.8' @@ -43,7 +43,7 @@ jobs: continue-on-error: true name: Check isort rules steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: '3.8' @@ -56,17 +56,20 @@ jobs: continue-on-error: true strategy: matrix: - version: [22.04, 20.04, docker] + version: ['24.04', '22.04', '20.04', docker] include: - - version: 22.04 + - version: '24.04' + distro: noble + - version: '22.04' distro: jammy - - version: 20.04 + - version: '20.04' distro: focal - version: docker distro: docker image + name: Build ${{ matrix.distro }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Prepare debian versioning if: ${{ matrix.version != 'docker' }} run: | @@ -81,7 +84,7 @@ jobs: - name: Archive package artifact if: ${{ matrix.version != 'docker' }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: debian-${{ matrix.distro }} path: | @@ -99,21 +102,23 @@ jobs: - name: Upload image if: ${{ matrix.version == 'docker' }} - uses: ishworkh/docker-image-artifact-upload@v1 + uses: ishworkh/container-image-artifact-upload@v2.0.0 with: image: "convertit:latest" e2e: - continue-on-error: true strategy: matrix: - distro: [ jammy, focal, docker ] + distro: [ noble, jammy, focal, docker ] include: + - distro: noble + version: '24.04' + - distro: jammy - version: 22.04 + version: '22.04' - distro: focal - version: 20.04 + version: '20.04' - distro: docker version: latest @@ -121,15 +126,16 @@ jobs: name: Test ${{ matrix.distro }} runs-on: ubuntu-${{ matrix.version }} needs: [ build ] + continue-on-error: true steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 if: ${{ matrix.distro != 'docker' }} with: name: debian-${{ matrix.distro }} - name: Download image if: ${{ matrix.distro == 'docker' }} - uses: ishworkh/docker-image-artifact-download@v1 + uses: ishworkh/container-image-artifact-download@v2.0.0 with: image: "convertit:latest" @@ -146,7 +152,7 @@ jobs: - name: Test ODT / PDF conversion run: | - wget -O myfile.pdf "http://localhost:6543/?url=https://file-examples.com/storage/febf69dcf3656dfd992b0fa/2017/10/file-sample_100kB.odt&from=application/vnd.oasis.opendocument.text" + wget -O myfile.pdf "http://localhost:6543/?url=https://file-examples.com/wp-content/storage/2017/10/file-sample_100kB.odt&from=application/vnd.oasis.opendocument.text" if file -b --mime-type ./myfile.pdf |grep -q 'application/pdf'; then echo "File is PDF"; @@ -159,7 +165,7 @@ jobs: - name: Test ODT / DOC conversion run: | - wget -O myfile.doc "http://localhost:6543/?url=https://file-examples.com/storage/febf69dcf3656dfd992b0fa/2017/10/file-sample_100kB.odt&from=application/vnd.oasis.opendocument.text&to=application/msword" + wget -O myfile.doc "http://localhost:6543/?url=https://file-examples.com/storage/fef4e75e176737761a179bf/2017/10/file-sample_100kB.odt&from=application/vnd.oasis.opendocument.text&to=application/msword" if file -b --mime-type ./myfile.doc |grep -q 'application/msword'; then echo "File is DOC"; @@ -195,25 +201,32 @@ jobs: file -b -f ./myfile.pdf; exit 1; fi + deploy: name: Publish (on release only) runs-on: ubuntu-latest needs: [ flake8, isort, e2e ] if: ${{ github.event_name == 'release' && github.event.action == 'created' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + + - name: Download 24.04 debian artifact + uses: actions/download-artifact@v4 + with: + name: debian-noble + - name: Download 22.04 debian artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: debian-jammy - name: Download 20.04 debian artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: debian-focal - name: Download docker image - uses: ishworkh/docker-image-artifact-download@v1 + uses: ishworkh/container-image-artifact-download@v2.0.0 with: image: "convertit:latest" diff --git a/Dockerfile b/Dockerfile index 039528b..1375fb9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ ARG DISTRO=jammy -FROM ubuntu:${DISTRO} as base -MAINTAINER Makina Corpus "contact@makina-corpus.com" +FROM ubuntu:${DISTRO} AS base +LABEL org.opencontainers.image.authors="Makina Corpus " RUN apt-get update && apt-get install -y -qq python3 libreoffice default-jre libreoffice-java-common inkscape python3-magic && \ apt-get autoclean && apt-get clean all && rm -rf /var/apt/lists/* @@ -9,21 +9,34 @@ RUN apt-get update && apt-get install -y -qq python3 libreoffice default-jre lib WORKDIR /opt/apps/convertit -ADD .docker/run.sh /usr/local/bin/run +COPY .docker/run.sh /usr/local/bin/run EXPOSE 6543 CMD ["/bin/sh", "-e", "/usr/local/bin/run"] -FROM base as build +FROM base AS build RUN apt-get update && apt-get install -y -qq build-essential python3-venv python3-dev && \ apt-get autoclean && apt-get clean all && rm -rf /var/apt/lists/* -ADD . /opt/apps/convertit +COPY requirements.txt /requirements.txt -RUN python3 -m venv . && ./bin/pip install --no-cache-dir pip setuptools wheel -U -RUN ./bin/python setup.py install +RUN python3 -m venv /opt/venv && /opt/venv/bin/pip install --no-cache-dir pip setuptools wheel -U && /opt/venv/bin/pip install --no-cache-dir -U -r /requirements.txt -FROM base as prod + +FROM build AS dev + +COPY dev-requirements.txt /dev-requirements.txt + +RUN /opt/venv/bin/pip install --no-cache-dir -U -r /dev-requirements.txt + +FROM base AS prod COPY --from=build /opt/apps/convertit /opt/apps/convertit -ADD .docker/run.sh /usr/local/bin/run +COPY --from=build /opt/venv /opt/venv +COPY convertit /opt/apps/convertit/convertit +COPY setup.py /opt/apps/convertit/setup.py +COPY README.rst /opt/apps/convertit/README.rst +RUN /opt/venv/bin/pip install . +COPY production.ini /opt/apps/convertit/production.ini +VOLUME /var/cache/convertit/downloads +VOLUME /var/cache/convertit/converted \ No newline at end of file diff --git a/convertit/converters/unoconv.py b/convertit/converters/unoconv.py index 567bc47..ebd2ac1 100644 --- a/convertit/converters/unoconv.py +++ b/convertit/converters/unoconv.py @@ -3,7 +3,6 @@ import subprocess import tempfile from functools import partial -from mimetypes import types_map from convertit import exists @@ -68,9 +67,9 @@ def is_available(): def converters(): return { - (types_map['.csv'], types_map['.ods']): to_ods, - (types_map['.csv'], types_map['.xls']): to_xls, - (types_map['.ods'], types_map['.xls']): to_xls, - (types_map['.odt'], types_map['.doc']): to_doc, - (types_map['.odt'], types_map['.pdf']): to_pdf, + ('plain/text', 'application/vnd.oasis.opendocument.spreadsheet'): to_ods, + ('plain/text', 'application/vnd.ms-excel'): to_xls, + ('application/vnd.oasis.opendocument.spreadsheet', 'application/vnd.ms-excel'): to_xls, + ('application/vnd.oasis.opendocument.text', 'application/msword'): to_doc, + ('application/vnd.oasis.opendocument.text', 'application/pdf'): to_pdf, } diff --git a/debian/control b/debian/control index 79abde3..d4fd300 100644 --- a/debian/control +++ b/debian/control @@ -16,6 +16,7 @@ Package: convertit Architecture: any Depends: ${misc:Depends}, + ${shlibs:Depends}, python3, python3-magic, inkscape, diff --git a/debian/postinst b/debian/postinst index 526bbfb..8df90a1 100644 --- a/debian/postinst +++ b/debian/postinst @@ -2,6 +2,6 @@ adduser --system --group --home /run/convertit --no-create-home --quiet convertit || true mkdir -p /var/cache/convertit || true -chown convertit.convertit /var/cache/convertit || true +chown convertit:convertit /var/cache/convertit || true #DEBHELPER# diff --git a/debian/rules b/debian/rules index acc21a3..98664f9 100755 --- a/debian/rules +++ b/debian/rules @@ -9,7 +9,11 @@ override_dh_virtualenv: dh_virtualenv \ --python /usr/bin/python3 \ --extra-pip-arg --no-cache-dir \ + --upgrade-pip \ + --preinstall wheel \ + --preinstall setuptools \ --builtin-venv + mkdir -p debian/convertit/usr/bin mkdir -p debian/convertit/opt/convertit cp production.ini debian/convertit/opt/convertit/convertit.ini diff --git a/dev-requirements.in b/dev-requirements.in new file mode 100644 index 0000000..d2843a7 --- /dev/null +++ b/dev-requirements.in @@ -0,0 +1,9 @@ +-c requirements.txt + +webtest +mock +coverage +freezegun +flake8 +isort +pip-tools \ No newline at end of file diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..b8bbb84 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,56 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile dev-requirements.in +# +beautifulsoup4==4.12.2 + # via webtest +build==1.0.3 + # via pip-tools +click==8.1.7 + # via pip-tools +coverage==7.3.2 + # via -r dev-requirements.in +flake8==6.1.0 + # via -r dev-requirements.in +freezegun==1.3.1 + # via -r dev-requirements.in +isort==5.12.0 + # via -r dev-requirements.in +mccabe==0.7.0 + # via flake8 +mock==5.1.0 + # via -r dev-requirements.in +packaging==23.2 + # via + # -c requirements.txt + # build +pip-tools==7.3.0 + # via -r dev-requirements.in +pycodestyle==2.11.1 + # via flake8 +pyflakes==3.1.0 + # via flake8 +pyproject-hooks==1.0.0 + # via build +python-dateutil==2.8.2 + # via freezegun +six==1.16.0 + # via python-dateutil +soupsieve==2.5 + # via beautifulsoup4 +waitress==2.1.2 + # via webtest +webob==1.8.7 + # via + # -c requirements.txt + # webtest +webtest==3.0.0 + # via -r dev-requirements.in +wheel==0.42.0 + # via pip-tools + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..39eccb6 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,7 @@ +version: "3.8" +services: + web: + build: + target: dev + volumes: + - .:/opt/apps/convertit \ No newline at end of file diff --git a/production.ini b/production.ini index f0a9eee..8ddac84 100644 --- a/production.ini +++ b/production.ini @@ -35,7 +35,7 @@ convertit.converters = [server:main] use = egg:gunicorn#main -host = localhost +host = 0.0.0.0 port = 6543 # Begin logging configuration diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..5b72317 --- /dev/null +++ b/requirements.in @@ -0,0 +1,4 @@ +pyramid +python-magic +Pillow +gunicorn \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..daad2c6 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,39 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile requirements.in +# +gunicorn==23.0.0 + # via -r requirements.in +hupper==1.12 + # via pyramid +packaging==23.2 + # via gunicorn +pastedeploy==3.1.0 + # via plaster-pastedeploy +pillow==10.1.0 + # via -r requirements.in +plaster==1.1.2 + # via + # plaster-pastedeploy + # pyramid +plaster-pastedeploy==1.0.1 + # via pyramid +pyramid==2.0.2 + # via -r requirements.in +python-magic==0.4.27 + # via -r requirements.in +translationstring==1.4 + # via pyramid +venusian==3.1.0 + # via pyramid +webob==1.8.7 + # via pyramid +zope-deprecation==5.0 + # via pyramid +zope-interface==6.1 + # via pyramid + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/setup.py b/setup.py index edfd39a..3e1fad2 100644 --- a/setup.py +++ b/setup.py @@ -4,29 +4,11 @@ here = os.path.abspath(os.path.dirname(__file__)) README = open(os.path.join(here, 'README.rst'), encoding='utf-8').read() -CHANGES = open(os.path.join(here, 'CHANGES.rst'), encoding='utf-8').read() - -requires = [ - 'pyramid<=1.10.5', - 'gunicorn', - 'python-magic', - 'Pillow', -] - -test_requires = requires + [ - 'webtest', - 'mock', - 'coverage', - 'freezegun', - 'flake8', - 'isort' -] - setup(name='convertit', version='2.2.6.dev0', description='A file conversion Web API in Pyramid', - long_description=README + '\n\n' + CHANGES, + long_description=README, license='AGPLV3', classifiers=[ "License :: OSI Approved :: GNU Affero General Public License v3", @@ -42,11 +24,6 @@ packages=find_packages(), include_package_data=True, zip_safe=False, - install_requires=requires, - tests_require=test_requires, - extras_require={ - 'test': test_requires, - }, test_suite="convertit", entry_points="""\ [paste.app_factory]