blacklight/platypush
1
0
Fork 0
forked from platypush/platypush
Code Issues Pull requests Projects Releases Packages Wiki Activity

[CI/CD] Do all the apt repo operations on a temporary folder.

Browse source 
`update-apt-repo` should first copy the existing repo root on the S3
bucket to a local temporary directory, so we don't do expensive (and
very slow) file operations on the s3fs filesystem.

After everything is done, we should do a current->old, tmp->current, rm old
dance to update the repo on S3.
This commit is contained in:
Fabio Manganiello 2023-10-14 15:14:08 +02:00
parent 465ff87f15
commit 1649808efc
Signed by: blacklight
GPG key ID: D90FBA7F76362774
1 changed files with 21 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
.drone.yml
View file

@ -568,10 +568,15 @@ steps:
- mkdir -p "$BUCKET_MNT" - mkdir -p "$BUCKET_MNT"
- s3fs "$S3_BUCKET" "$BUCKET_MNT" -o url="$AWS_ENDPOINT_URL" - s3fs "$S3_BUCKET" "$BUCKET_MNT" -o url="$AWS_ENDPOINT_URL"
- export APT_ROOT="$BUCKET_MNT/apt" - export APT_ROOT="$BUCKET_MNT/apt"
- mkdir -p "$APT_ROOT"
- echo "-- Creating a new apt root folder"
- export TMP_APT_ROOT="/tmp/apt"
- cp -r "$APT_ROOT" "$TMP_APT_ROOT"
- echo "-- Cleaning up older apt releases" - echo "-- Cleaning up older apt releases"
- | - |
find "$APT_ROOT/pool" -mindepth 2 -maxdepth 2 -type d | while read reldir; do find "$TMP_APT_ROOT/pool" -mindepth 2 -maxdepth 2 -type d | while read reldir; do
pkg_to_remove=$(( $(ls "$reldir"/*.deb | wc -l) - 1 )) pkg_to_remove=$(( $(ls "$reldir"/*.deb | wc -l) - 1 ))
[ $pkg_to_remove -le 0 ] && continue [ $pkg_to_remove -le 0 ] && continue
ls "$reldir"/*.deb | sort -V | head -n$pkg_to_remove | xargs rm -f ls "$reldir"/*.deb | sort -V | head -n$pkg_to_remove | xargs rm -f
@ -581,13 +586,13 @@ steps:
- | - |
echo "stable\noldstable" | while read distro; do echo "stable\noldstable" | while read distro; do
echo "main\ndev" | while read branch; do echo "main\ndev" | while read branch; do
branch_dir="$APT_ROOT/pool/$distro/$branch" branch_dir="$TMP_APT_ROOT/pool/$distro/$branch"
[ -d "$branch_dir" ] || continue [ -d "$branch_dir" ] || continue
dist_dir="$APT_ROOT/dists/$distro/$branch/all" dist_dir="$TMP_APT_ROOT/dists/$distro/$branch/all"
mkdir -p "$dist_dir" mkdir -p "$dist_dir"
pkg_file="$dist_dir/Packages" pkg_file="$dist_dir/Packages"
dpkg-scanpackages --arch all "$branch_dir" > "$pkg_file" dpkg-scanpackages --arch all "$branch_dir" > "$pkg_file"
sed -i "$pkg_file" -re "s|^Filename: $APT_ROOT/|Filename: |" sed -i "$pkg_file" -re "s|^Filename: $TMP_APT_ROOT/|Filename: |"
cat "$pkg_file" | gzip -9 > "$pkg_file.gz" cat "$pkg_file" | gzip -9 > "$pkg_file.gz"
done done
done done
@ -614,7 +619,7 @@ steps:
} }
echo "stable\noldstable" | while read distro; do echo "stable\noldstable" | while read distro; do
dist_dir="$APT_ROOT/dists/$distro" dist_dir="$TMP_APT_ROOT/dists/$distro"
components=$(find "$dist_dir" -name Packages | awk -F '/' '{print $(NF-2)}' | uniq | tr '\n' ' ') components=$(find "$dist_dir" -name Packages | awk -F '/' '{print $(NF-2)}' | uniq | tr '\n' ' ')
release_file="$dist_dir/Release" release_file="$dist_dir/Release"
@ -635,7 +640,7 @@ steps:
done done
- echo "-- Importing and refreshing PGP key" - echo "-- Importing and refreshing PGP key"
- echo "$PGP_PUBKEY" > "$APT_ROOT/pubkey.txt" - echo "$PGP_PUBKEY" > "$TMP_APT_ROOT/pubkey.txt"
- export PGP_KEYID=$(echo "$PGP_PUBKEY" | gpg --with-colons --import-options show-only --import --fingerprint | grep -e '^fpr:' | head -1 | awk -F ':' '{print $(NF - 1)}') - export PGP_KEYID=$(echo "$PGP_PUBKEY" | gpg --with-colons --import-options show-only --import --fingerprint | grep -e '^fpr:' | head -1 | awk -F ':' '{print $(NF - 1)}')
- | - |
cat <<EOF | gpg --import --armor cat <<EOF | gpg --import --armor
@ -644,12 +649,21 @@ steps:
- echo "-- Signing Release files" - echo "-- Signing Release files"
- | - |
find "$APT_ROOT/dists" -type f -name Release | while read file; do find "$TMP_APT_ROOT/dists" -type f -name Release | while read file; do
dirname="$(dirname "$file")" dirname="$(dirname "$file")"
cat "$file" | gpg -q --default-key "$PGP_KEYID" -abs > "$file.gpg" cat "$file" | gpg -q --default-key "$PGP_KEYID" -abs > "$file.gpg"
cat "$file" | gpg -q --default-key "$PGP_KEYID" -abs --clearsign > "$dirname/InRelease" cat "$file" | gpg -q --default-key "$PGP_KEYID" -abs --clearsign > "$dirname/InRelease"
done done
- echo "-- Updating the apt repo root on S3"
- export OLD_APT_ROOT="$BUCKET_MNT/oldapt"
- |
if [ ! -d "$OLD_APT_ROOT" ]; then
mv "$APT_ROOT" "$OLD_APT_ROOT"
mv "$TMP_APT_ROOT" "$APT_ROOT"
rm -rf "$OLD_APT_ROOT"
fi
### ###
### Update the RPM (stable) packages ### Update the RPM (stable) packages
### ###