CI: fix VVS & Entur smoke tests 💚

Author: derhuerst

.github/workflows/smoke-test.yml

@@ -34,6 +34,11 @@ jobs:
     - name: install sponge (moreutils)
       run: sudo apt install -y moreutils
 
+    - name: install QSV
+      run: ./ci/install-qsv.sh
+    - name: enable QSV for file extensions other than .qsv
+      run: |
+        echo 'QSV_SKIP_FORMAT_CHECK=true' >>$GITHUB_ENV
     - name: install DuckDB
       run: |
         set -eux -o pipefail

.github/workflows/smoke-tests.yml

@@ -21,7 +21,7 @@ jobs:
     with:
       feed-name: Entur
       feed-download-cmd: 'wget -nv -U "${{ github.repository }} CI" "https://storage.googleapis.com/marduk-production/outbound/gtfs/rb_norway-aggregated-gtfs.zip" -O gtfs.zip'
-      feed-extract-cmd: 'unzip -d gtfs gtfs.zip'
+      feed-extract-cmd: 'unzip -d gtfs gtfs.zip && qsv select "!feed_id" gtfs/feed_info.txt | sponge gtfs/feed_info.txt'
       feed-import-flags: '--route-types-scheme tpeg-pti'
       feed-import-glob: 'gtfs/*.txt'
   smoke-test-vgn:
@@ -39,6 +39,6 @@ jobs:
     with:
       feed-name: VVS
       feed-download-cmd: 'wget -nv -U "${{ github.repository }} CI" "https://download.vvs.de/gtfs_realtime.zip" -O gtfs.zip'
-      feed-extract-cmd: 'unzip -d gtfs gtfs.zip && rm gtfs/ticketing_deep_links.txt gtfs/ticketing_identifiers.txt'
+      feed-extract-cmd: 'unzip -d gtfs gtfs.zip && rm gtfs/ticketing_deep_links.txt gtfs/ticketing_identifiers.txt && qsv select "!ticketing_deep_link_id" gtfs/agency.txt | sponge gtfs/agency.txt'
       feed-import-flags: '--trips-without-shape-id --lower-case-lang-codes'
       feed-import-glob: 'gtfs/*.txt'

ci/install-qsv.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -eu -o pipefail
+set -x
+
+# alternative: cargo install qsv --locked --features all_features
+
+sudo apt install -y \
+	curl jq unzip
+
+ua='iris-gtfs-rt-feed CI'
+qsv_release='286558142' # v16.1.0
+releases_url="https://api.github.com/repos/dathere/qsv/releases/$qsv_release"
+assets_url="$(
+	curl "$releases_url" -H 'Accept: application/json' -H "User-Agent: $ua" -L -fsS \
+	| jq -r '.assets_url'
+)"
+qsv_x64_url="$(
+	curl "$assets_url" -H 'Accept: application/json' -H "User-Agent: $ua" -L -fsS \
+	| jq -r '.[] | select(.name | test("qsv-[0-9.]+-x86_64-unknown-linux-gnu.zip")) | .browser_download_url'
+)"
+
+zip="$(mktemp)"
+curl -o "$zip" "$qsv_x64_url" -H "User-Agent: $ua" -L -fsS
+
+mkdir -p "$HOME/.local/bin"
+unzip -j -q -d "$HOME/.local/bin" "$zip" qsv
+chmod +x "$HOME/.local/bin/qsv"
+
+$HOME/.local/bin/qsv --version