ci: test

.github/workflows/data-pipeline.yml

@@ -0,0 +1,21 @@
+# SPDX-FileCopyrightText: 2024 Jonah Brüchert <[email protected]>
+# SPDX-License-Identifier: CC0-1.0
+
+name: Import data
+
+on: [push, workflow_dispatch]
+
+jobs:
+  import:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v3
+    - name: Fetch submodules
+      run: git submodule update --init --checkout --remote
+    - name: Build docker images
+      run: docker build -t transitous . -f ci/container/Containerfile
+    - name: Fetch GTFS-Feeds
+      run: docker run -v $PWD:/transitous -w /transitous transitous ci/fetch-feeds.py timer
+    - name: Generate MOTIS config file
+      run: docker run --env PYTHONPATH=src -v $PWD:/transitous -w /transitous transitous ci/generate-motis-config.py

.github/workflows/test-import.yml

@@ -0,0 +1,19 @@
+# SPDX-FileCopyrightText: 2024 Jonah Brüchert <[email protected]>
+# SPDX-License-Identifier: CC0-1.0
+
+name: Verfy new data can be imported
+
+on: [pull-request]
+
+jobs:
+  import:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v3
+    - name: Fetch submodules
+      run: git submodule update --init --checkout --remote
+    - name: Build docker images
+      run: docker build -t transitous . -f ci/container/Containerfile
+    - name: Verify that new feeds can be downloaded
+      run: docker run -v $PWD:/transitous -w /transitous transitous ci/fetch-feeds.py merge-request

ci/container/Containerfile

@@ -1,9 +1,9 @@
+# SPDX-FileCopyrightText: 2024 Jonah Brüchert <[email protected]>
+# SPDX-License-Identifier: CC0-1.0
+
 FROM docker.io/debian:bookworm-slim
 
-RUN apt-get update && apt-get install git python3-requests golang -y && apt clean
+RUN apt-get update && apt-get install git python3-requests python3-jinja2 golang -y && apt clean
 
 ADD https://github.com/motis-project/motis/releases/download/v0.11.18/motis-linux-amd64.tar.bz2 /opt/
 RUN GOBIN=/usr/local/bin/ go install github.com/patrickbr/gtfstidy@latest
-
-RUN adduser motis
-USER motis

src/fetch.py

@@ -68,16 +68,20 @@ def fetch(self, metadata: Path):
         metadata_filename = metadata.name
         region_name = metadata_filename[:metadata_filename.rfind('.')]
 
-        print(f"Fetching {region_name}…")
-
         for source in region.sources:
             download_name = f"{region_name}_{source.name}"
 
+            print(f"Fetching {region_name}-{source.name}…")
+            sys.stdout.flush()
             dest_path = self.fetch_source(download_name, source)
 
-            print(f"Postproccing {region_name} with gtfstidy…")
+            print(f"Postproccing {region_name}-{source.name} with gtfstidy…")
+            sys.stdout.flush()
             self.postprocess(source, dest_path)
 
+            print()
+            sys.stdout.flush()
+
 
 if __name__ == "__main__":
     fetcher = Fetcher()