From 6fb432d7bb6e5d83c06fc89e8e043fa83b66fbdb Mon Sep 17 00:00:00 2001
From: Fabio Manganiello <fabio@manganiello.tech>
Date: Thu, 19 Oct 2023 21:48:59 +0200
Subject: [PATCH] [CI/CD] S3 bucket replaced with a local folder.
Managing concurrent copies over s3fs through CI/CD is a hassle.
Input/output errors abound and the steps are also slow.
---
.drone.yml | 101 ++++++++++++++++++++---------------------------------
1 file changed, 38 insertions(+), 63 deletions(-)
diff --git a/.drone.yml b/.drone.yml
index 19465dc77..8f22f9f02 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -8,6 +8,10 @@ volumes:
host:
path: /opt/docs/platypush
+ - name: repos
+ host:
+ path: /opt/repos/platypush
+
steps:
###
@@ -330,19 +334,15 @@ steps:
- name: update-debian-stable-packages
image: debian:stable
- privileged: true # Required to use the FUSE module for s3fs
+ volumes:
+ - name: repos
+ path: /repos
+
environment:
DEB_VERSION: stable
WORKDIR: /tmp/workdir
+ APT_ROOT: /repos/apt
PKG_NAME: platypush
- S3_BUCKET: platypush-pkg
- BUCKET_MNT: /mnt/s3
- AWS_ENDPOINT_URL: https://s3.nl-ams.scw.cloud
- AWS_DEFAULT_REGION: nl-ams
- AWS_ACCESS_KEY_ID:
- from_secret: aws_access_key_id
- AWS_SECRET_ACCESS_KEY:
- from_secret: aws_secret_access_key
when:
branch:
@@ -356,7 +356,7 @@ steps:
commands:
- echo "-- Installing dependencies"
- apt update
- - apt install -y curl dpkg-dev gpg git python3 python3-pip s3fs
+ - apt install -y curl dpkg-dev gpg git python3 python3-pip
- echo "-- Copying source directory"
- mkdir -p "$WORKDIR"
@@ -393,12 +393,7 @@ steps:
EOF
- dpkg --build "$GIT_BUILD_DIR"
- - echo "-- Mounting the S3 bucket"
- - mkdir -p "$BUCKET_MNT"
- - s3fs "$S3_BUCKET" "$BUCKET_MNT" -o url="$AWS_ENDPOINT_URL" -o nomixupload
- - export APT_ROOT="$BUCKET_MNT/apt"
-
- - echo "--- Pushing package to the S3 bucket"
+ - echo "--- Copying package to the repo"
- mkdir -p "$APT_ROOT/pool/$DEB_VERSION/dev"
- cp "$GIT_DEB" "$APT_ROOT/pool/$DEB_VERSION/dev/"
@@ -432,17 +427,14 @@ steps:
- name: update-debian-oldstable-packages
image: debian:oldstable
+ volumes:
+ - name: repos
+ path: /repos
+
environment:
DEB_VERSION: oldstable
WORKDIR: /tmp/workdir
- PKG_NAME: platypush
- S3_URL: s3://platypush-pkg/apt
- AWS_ENDPOINT_URL: https://s3.nl-ams.scw.cloud
- AWS_DEFAULT_REGION: nl-ams
- AWS_ACCESS_KEY_ID:
- from_secret: aws_access_key_id
- AWS_SECRET_ACCESS_KEY:
- from_secret: aws_secret_access_key
+ APT_ROOT: /repos/apt
when:
branch:
@@ -463,7 +455,6 @@ steps:
- echo "-- Installing dependencies"
- apt update
- apt install -y curl dpkg-dev gpg git python3 python3-pip
- - pip install awscli
- echo "--- Parsing metadata"
- git config --global --add safe.directory $PWD
@@ -494,8 +485,9 @@ steps:
EOF
- dpkg --build "$GIT_BUILD_DIR"
- - echo "--- Pushing package to the S3 bucket"
- - aws s3 cp "$GIT_DEB" "$S3_URL/pool/$DEB_VERSION/dev/"
+ - echo "--- Copying package to the repo"
+ - mkdir -p "$APT_ROOT/pool/$DEB_VERSION/dev"
+ - cp "$GIT_DEB" "$APT_ROOT/pool/$DEB_VERSION/dev/"
# If main/all/Packages doesn't exist, then we should create the first main release
- export PKGURL="https://apt.platypush.tech/dists/$DEB_VERSION/main/all/Packages"
@@ -527,16 +519,13 @@ steps:
- name: update-apt-repo
image: debian:stable
- privileged: true # Required to use the FUSE module for s3fs
+ volumes:
+ - name: repos
+ path: /repos
+
environment:
- S3_BUCKET: platypush-pkg
- BUCKET_MNT: /mnt/s3
- AWS_ENDPOINT_URL: https://s3.nl-ams.scw.cloud
- AWS_DEFAULT_REGION: nl-ams
- AWS_ACCESS_KEY_ID:
- from_secret: aws_access_key_id
- AWS_SECRET_ACCESS_KEY:
- from_secret: aws_secret_access_key
+ REPOS_ROOT: /repos
+ APT_ROOT: /repos/apt
PGP_PUBKEY:
from_secret: apt_pgp_pub_key
PGP_PRIVKEY:
@@ -555,13 +544,7 @@ steps:
commands:
- echo "-- Installing dependencies"
- apt update
- - apt install -y s3fs dpkg-dev gpg
-
- - echo "-- Mounting the S3 bucket"
- - mkdir -p "$BUCKET_MNT"
- - s3fs "$S3_BUCKET" "$BUCKET_MNT" -o url="$AWS_ENDPOINT_URL" -o nomixupload
- - export APT_ROOT="$BUCKET_MNT/apt"
- - mkdir -p "$APT_ROOT"
+ - apt install -y dpkg-dev gpg
- echo "-- Creating a new apt root folder"
- export TMP_APT_ROOT="/tmp/apt"
@@ -648,8 +631,8 @@ steps:
cat "$file" | gpg -q --default-key "$PGP_KEYID" -abs --clearsign > "$dirname/InRelease"
done
- - echo "-- Updating the apt repo root on S3"
- - export OLD_APT_ROOT="$BUCKET_MNT/oldapt"
+ - echo "-- Updating the apt repo root"
+ - export OLD_APT_ROOT="$REPOS_ROOT/oldapt"
- |
if [ ! -d "$OLD_APT_ROOT" ]; then
mv "$APT_ROOT" "$OLD_APT_ROOT"
@@ -663,19 +646,16 @@ steps:
- name: update-rpm-repo
image: fedora
- privileged: true # Required to use the FUSE module for s3fs
+ volumes:
+ - name: repos
+ path: /repos
+
environment:
RPM_VERSION: stable
+ REPOS_ROOT: /repos
+ RPM_ROOT: /repos/rpm
WORKDIR: /tmp/workdir
PKG_NAME: platypush
- S3_BUCKET: platypush-pkg
- BUCKET_MNT: /mnt/s3
- AWS_ENDPOINT_URL: https://s3.nl-ams.scw.cloud
- AWS_DEFAULT_REGION: nl-ams
- AWS_ACCESS_KEY_ID:
- from_secret: aws_access_key_id
- AWS_SECRET_ACCESS_KEY:
- from_secret: aws_secret_access_key
PGP_PUBKEY:
from_secret: rpm_pgp_pub_key
PGP_PRIVKEY:
@@ -692,18 +672,13 @@ steps:
commands:
- echo "-- Installing dependencies"
- - yum install -y createrepo rpm-build rpm-sign gpg wget yum-utils git python python-pip s3fs-fuse
+ - yum install -y createrepo rpm-build rpm-sign gpg wget yum-utils git python python-pip
- echo "-- Copying source directory"
- mkdir -p "$WORKDIR"
- export SRCDIR="$WORKDIR/src"
- cp -r "$PWD" "$SRCDIR"
- cd "$SRCDIR"
-
- - echo "-- Mounting the S3 bucket"
- - mkdir -p "$BUCKET_MNT"
- - s3fs "$S3_BUCKET" "$BUCKET_MNT" -o url="$AWS_ENDPOINT_URL" -o nomixupload
- - export RPM_ROOT="$BUCKET_MNT/rpm"
- mkdir -p "$RPM_ROOT"
- echo "--- Parsing metadata"
@@ -846,9 +821,9 @@ steps:
- createrepo "$TMP_RPM_ROOT"
- gpg --detach-sign --armor "$TMP_RPM_ROOT/repodata/repomd.xml"
- - echo "--- Updating the S3 bucket"
- - export NEW_RPM_ROOT="$BUCKET_MNT/rpm_new"
- - export OLD_RPM_ROOT="$BUCKET_MNT/rpm_old"
+ - echo "--- Updating the repository"
+ - export NEW_RPM_ROOT="$REPOS_ROOT/rpm_new"
+ - export OLD_RPM_ROOT="$REPOS_ROOT/rpm_old"
- cp -r "$TMP_RPM_ROOT" "$NEW_RPM_ROOT"
- rm -rf "$TMP_RPM_ROOT"
- mv "$RPM_ROOT" "$OLD_RPM_ROOT"