From 7fa0dbda7bdfe8e082bf87b6e6ecef89ab7c2cde Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Thu, 23 May 2024 00:17:55 +0200 Subject: [PATCH] Split Drone CI steps into separate shell scripts. A fully self-contained 1.5k LoC Drone file isn't very maintainable, and it makes it hard to reuse parts that are shared across multiple steps (like SSH and git configuration). --- .drone.yml | 816 +----------------------------- .drone/build-ui.sh | 43 ++ .drone/github-mirror.sh | 13 + .drone/macros/configure-git.sh | 29 ++ .drone/macros/configure-gpg.sh | 30 ++ .drone/macros/configure-ssh.sh | 35 ++ .drone/rebuild-docs.sh | 28 + .drone/run-tests.sh | 6 + .drone/sync-stable-branch.sh | 18 + .drone/update-apt-repo.sh | 166 ++++++ .drone/update-arch-packages.sh | 67 +++ .drone/update-components-cache.sh | 46 ++ .drone/update-deb-packages.sh | 71 +++ .drone/update-pip-package.sh | 5 + .drone/update-rpm-repo.sh | 201 ++++++++ 15 files changed, 770 insertions(+), 804 deletions(-) create mode 100755 .drone/build-ui.sh create mode 100755 .drone/github-mirror.sh create mode 100755 .drone/macros/configure-git.sh create mode 100755 .drone/macros/configure-gpg.sh create mode 100755 .drone/macros/configure-ssh.sh create mode 100755 .drone/rebuild-docs.sh create mode 100755 .drone/run-tests.sh create mode 100755 .drone/sync-stable-branch.sh create mode 100755 .drone/update-apt-repo.sh create mode 100755 .drone/update-arch-packages.sh create mode 100755 .drone/update-components-cache.sh create mode 100755 .drone/update-deb-packages.sh create mode 100755 .drone/update-pip-package.sh create mode 100755 .drone/update-rpm-repo.sh diff --git a/.drone.yml b/.drone.yml index dfba36ec5..bf048d47b 100644 --- a/.drone.yml +++ b/.drone.yml @@ -27,19 +27,7 @@ steps: from_secret: ssh_privkey commands: - - apk add --update --no-cache git openssh - - mkdir -p ~/.ssh - - echo $SSH_PUBKEY > ~/.ssh/id_rsa.pub - - | - cat < ~/.ssh/id_rsa - $SSH_PRIVKEY - EOF - - chmod 0600 ~/.ssh/id_rsa - - ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null - - git config --global --add safe.directory $PWD - - git remote add github git@github.com:/BlackLight/platypush.git - - git pull --rebase github "$(git branch | head -1 | awk '{print $2}')" || echo "No such branch on Github" - - git push --all -v github + - . .drone/github-mirror.sh ### ### Synchronize the `stable` branch to the new release @@ -58,45 +46,7 @@ steps: from_secret: pgp_key_id commands: - - apk add --update --no-cache git bash gnupg openssh - - # Backup the original git configuration before changing attributes - - export GIT_CONF=$PWD/.git/config - - export TMP_GIT_CONF=/tmp/git.config.orig - - cp $GIT_CONF $TMP_GIT_CONF - - git config --global --add safe.directory $PWD - - # Install the SSH and PGP keys - - mkdir -p ~/.ssh - - | - cat < ~/.ssh/id_rsa.pub - - | - cat < ~/.ssh/id_rsa - $SSH_PRIVKEY - EOF - - chmod 0600 ~/.ssh/id_rsa - - ssh-keyscan git.platypush.tech >> ~/.ssh/known_hosts 2>/dev/null - - # Git configuration - - git config user.name "Platypush CI/CD Automation" - - git config user.email "admin@platypush.tech" - - git config commit.gpgsign true - - git config user.signingkey $PGP_KEY_ID - - git remote rm origin - - git remote add origin git@git.platypush.tech:platypush/platypush.git - - # Merge and push to the `stable` branch - - git checkout stable - - git rebase master - - git push -u origin stable - - git checkout master - - # Restore the original git configuration - - mv $TMP_GIT_CONF $GIT_CONF + - . .drone/sync-stable-branch.sh when: event: @@ -113,28 +63,7 @@ steps: path: /docs commands: - - echo "Installing required build dependencies" - - apk add --update --no-cache git make py3-sphinx py3-myst-parser py3-pip $(cat platypush/install/requirements/alpine.txt) - - pip install -U sphinx-rtd-theme sphinx-book-theme --break-system-packages - - pip install . --break-system-packages - - mkdir -p /docs/current - - export APPDIR=$PWD - - rm -rf "$APPDIR/docs/build" - - echo "Building the updated documentation" - - cd "$APPDIR/docs/source" - - git clone 'https://git.platypush.tech/platypush/platypush.wiki.git' wiki - - cd wiki - - ln -s Home.md index.md - - cd "$APPDIR/docs" - - make html - - rm -f config*.yaml - - cd "$APPDIR" - - echo "Copying the new documentation files to the target folder" - - mv -v "$APPDIR/docs/build" /docs/new - - cd /docs - - mv current old - - mv new current - - rm -rf old + - . .drone/rebuild-docs.sh when: event: @@ -150,10 +79,7 @@ steps: - name: tests image: alpine commands: - - apk add --update --no-cache $(cat platypush/install/requirements/alpine.txt) - - pip install . --break-system-packages - - pip install -r requirements-tests.txt --break-system-packages - - pytest tests + - . .drone/run-tests.sh ### ### Rebuild the UI files @@ -182,69 +108,7 @@ steps: - tests commands: - - export SKIPCI="$PWD/.skipci" - - rm -rf "$SKIPCI" - - apk add --update --no-cache git - - # Backup the original git configuration before changing attributes - - export GIT_CONF=$PWD/.git/config - - export TMP_GIT_CONF=/tmp/git.config.orig - - cp $GIT_CONF $TMP_GIT_CONF - - - git config --global --add safe.directory $PWD - - cd platypush/backend/http/webapp - - | - if [ $(git log --pretty=oneline $DRONE_COMMIT_AFTER...$DRONE_COMMIT_BEFORE . | wc -l) -eq 0 ]; then - echo "No UI changes detected, skipping build" - exit 0 - fi - - - | - if [ "$(git log --pretty=format:%s HEAD...HEAD~1 | head -1)" == "[Automatic] Updated UI files" ]; then - echo "UI changes have already been committed, skipping build" - exit 0 - fi - - - rm -rf dist node_modules - - npm install - - npm run build - - | - if [ $(git status --porcelain dist | wc -l) -eq 0 ]; then - echo "No build files have been changed" - exit 0 - fi - - # Create a .skipci file to mark the fact that the next steps should be skipped - # (we're going to do another push anyway, so another pipeline will be triggered) - - touch "$SKIPCI" - - - apk add --update --no-cache bash gnupg openssh - - mkdir -p ~/.ssh - - | - cat < ~/.ssh/id_rsa.pub - - | - cat < ~/.ssh/id_rsa - $SSH_PRIVKEY - EOF - - chmod 0600 ~/.ssh/id_rsa - - ssh-keyscan git.platypush.tech >> ~/.ssh/known_hosts 2>/dev/null - - - git config user.name "Platypush CI/CD Automation" - - git config user.email "admin@platypush.tech" - - git config commit.gpgsign true - - git config user.signingkey $PGP_KEY_ID - - git add dist - - git commit dist -S -m "[Automatic] Updated UI files" --no-verify - - git remote rm origin - - git remote add origin git@git.platypush.tech:platypush/platypush.git - - git push -f origin master - - # Restore the original git configuration - - mv $TMP_GIT_CONF $GIT_CONF + - . .drone/build-ui.sh ### ### Regenerate the components.json cache @@ -273,66 +137,7 @@ steps: - build-ui commands: - - export SKIPCI="$PWD/.skipci" - - export CACHEFILE="$PWD/platypush/components.json.gz" - - | - [ -f "$SKIPCI" ] && exit 0 - - # Only regenerate the components cache if either the plugins, backends, - # events or schemas folders have some changes (excluding the webapp files). - - apk add --update --no-cache git - - | - if [ -z "$(git log --pretty=oneline $DRONE_COMMIT_AFTER...$DRONE_COMMIT_BEFORE -- platypush/backend platypush/plugins platypush/schemas platypush/message/event ':(exclude)platypush/backend/http/webapp')" ]; then - echo 'No changes to the components file' - exit 0 - fi - - - echo 'Updating components cache' - - apk add --update --no-cache bash gnupg openssh $(cat platypush/install/requirements/alpine.txt) - - pip install . --break-system-packages - - | - python - < ~/.ssh/id_rsa.pub - - | - cat < ~/.ssh/id_rsa - $SSH_PRIVKEY - EOF - - chmod 0600 ~/.ssh/id_rsa - - ssh-keyscan git.platypush.tech >> ~/.ssh/known_hosts 2>/dev/null - - - git config user.name "Platypush CI/CD Automation" - - git config user.email "admin@platypush.tech" - - git config commit.gpgsign true - - git config user.signingkey $PGP_KEY_ID - - git add "$CACHEFILE" - - git commit "$CACHEFILE" -S -m "[Automatic] Updated components cache" --no-verify - - git remote rm origin - - git remote add origin git@git.platypush.tech:platypush/platypush.git - - git push -f origin master - - # Restore the original git configuration - - mv $TMP_GIT_CONF $GIT_CONF + - . .drone/update-components-cache.sh ### ### Update the Arch packages @@ -357,79 +162,7 @@ steps: - update-components-cache commands: - - | - [ -f .skipci ] && exit 0 - - - echo "-- Installing dependencies" - - apk add --update --no-cache curl git openssh pacman sudo - - - echo "--- Parsing metadata" - - git config --global --add safe.directory $PWD - - git pull --rebase origin master --tags - - export VERSION=$(python setup.py --version) - - export HEAD=$(git log --pretty=format:%h HEAD...HEAD~1 | head -1) - - export GIT_VERSION="$VERSION.r$(git log --pretty=oneline HEAD...v$VERSION | wc -l).g$${HEAD}" - - export TAG_URL="https://git.platypush.tech/platypush/platypush/archive/v$VERSION.tar.gz" - - - echo "--- Preparing environment" - - mkdir -p ~/.ssh - - echo $SSH_PUBKEY > ~/.ssh/id_rsa.pub - - | - cat < ~/.ssh/id_rsa - $SSH_PRIVKEY - EOF - - chmod 0600 ~/.ssh/id_rsa - - ssh-keyscan git.platypush.tech >> ~/.ssh/known_hosts 2>/dev/null - - ssh-keyscan aur.archlinux.org >> ~/.ssh/known_hosts 2>/dev/null - - adduser -u 1000 -D build - - mkdir -p $WORKDIR - - - echo "--- Updating Arch git version" - - export PKGDIR=$WORKDIR/git - - git clone ssh://aur@aur.archlinux.org/platypush-git.git $PKGDIR - - git config --global --add safe.directory $PKGDIR - - chown -R build $PKGDIR - - cd $PKGDIR - - | - sed -i 'PKGBUILD' -r \ - -e "s/^pkgver=.*/pkgver=$GIT_VERSION/" \ - -e "s/^pkgrel=.*/pkgrel=1/" \ - - sudo -u build makepkg --printsrcinfo > .SRCINFO - - export FILES_CHANGED=$(git status --porcelain --untracked-files=no | wc -l) - - | - if [ $FILES_CHANGED -gt 0 ]; then - echo "--- Pushing git package version $GIT_VERSION" - git commit -a -m '[Automatic] Package update' - git push origin master - fi - - - echo "--- Updating Arch stable version" - - export PKGDIR=$WORKDIR/stable - - git clone ssh://aur@aur.archlinux.org/platypush.git $PKGDIR - - git config --global --add safe.directory $PKGDIR - - chown -R build $PKGDIR - - cd $PKGDIR - - export RELEASED_VERSION=$(grep -e '^pkgver=' PKGBUILD | sed -r -e 's/^pkgver=(.*)\s*/\1/') - - | - if [ "$RELEASED_VERSION" == "$VERSION" ]; then - exit 0 - fi - - - export TAG_CHECKSUM=$(curl --silent "$TAG_URL" | sha512sum | awk '{print $1}') - - | - sed -i 'PKGBUILD' -r \ - -e "s/^pkgver=.*/pkgver=$VERSION/" \ - -e "s/^pkgrel=.*/pkgrel=1/" \ - -e "s/^sha512sums=.*/sha512sums=('$TAG_CHECKSUM')/" - - - sudo -u build makepkg --printsrcinfo > .SRCINFO - - export FILES_CHANGED=$(git status --porcelain --untracked-files=no | wc -l) - - | - if [ $FILES_CHANGED -gt 0 ]; then - echo "--- Pushing stable package version $VERSION" - git commit -a -m '[Automatic] Package update' - git push origin master - fi + - . .drone/update-arch-packages.sh ### ### Update the Debian (stable) packages @@ -457,78 +190,7 @@ steps: - update-components-cache commands: - - | - [ -f .skipci ] && exit 0 - - - echo "-- Installing dependencies" - - apt update - - apt install -y curl dpkg-dev gpg git python3 python3-pip python3-setuptools - - - echo "-- Copying source directory" - - mkdir -p "$WORKDIR" - - export SRCDIR="$WORKDIR/src" - - cp -r "$PWD" "$SRCDIR" - - cd "$SRCDIR" - - - echo "--- Parsing metadata" - - git config --global --add safe.directory "$PWD" - - git pull --rebase origin master --tags - - export VERSION=$(python3 setup.py --version) - - export GIT_VERSION="$VERSION-$(git log --pretty=oneline HEAD...v$VERSION | wc -l)" - - export GIT_BUILD_DIR="$WORKDIR/$${PKG_NAME}_$${GIT_VERSION}_all" - - export GIT_DEB="$WORKDIR/$${PKG_NAME}_$${GIT_VERSION}_all.deb" - - export POOL_PATH="$APT_ROOT/pool/$DEB_VERSION/dev" - - - echo "--- Building git package" - - pip install --prefix="$GIT_BUILD_DIR/usr" --no-cache --no-deps . - - | - find "$GIT_BUILD_DIR" -name "site-packages" | while read dir; do - base="$(dirname "$dir")" - mv "$dir" "$base/dist-packages" - done - - - mkdir -p "$GIT_BUILD_DIR/DEBIAN" - - | - cat < "$GIT_BUILD_DIR/DEBIAN/control" - Package: $PKG_NAME - Version: $GIT_VERSION - Maintainer: Fabio Manganiello - Depends: $(cat platypush/install/requirements/debian.txt | tr '\n' ',' | sed -re 's/,$//' -e 's/,/, /g') - Architecture: all - Homepage: https://platypush.tech - Description: Universal command executor and automation hub. - EOF - - mkdir -p "$POOL_PATH" - - rm -f "$POOL_PATH/"*.deb - - dpkg --build "$GIT_BUILD_DIR" - - - echo "--- Copying $GIT_DEB to $POOL_PATH" - - cp "$GIT_DEB" "$POOL_PATH" - - # If main/all/Packages doesn't exist, then we should create the first main release - - | - [ $(ls "$APT_ROOT/pool/$DEB_VERSION/main/$${PKG_NAME}_$${VERSION}-"*"_all.deb" 2>/dev/null | wc -l) -eq 0 ] && export UPDATE_STABLE_PKG=1 - - - export PKGURL="https://apt.platypush.tech/dists/$DEB_VERSION/main/all/Packages" - - | - [ -z "$UPDATE_STABLE_PKG" ] && - curl -ILs -o /dev/null -w "%{http_code}" "$PKGURL" | - grep -e '^4' >/dev/null && export UPDATE_STABLE_PKG=1 - - # If the published release version differs from the current one, then we should publish a new main release - - | - if [ -z "$UPDATE_STABLE_PKG" ]; then - RELEASED_VERSION=$(curl -s "$PKGURL" | grep -e '^Version: ' | head -1 | awk '{print $2}' | cut -d- -f 1) - [ "$RELEASED_VERSION" != "$VERSION" ] && export UPDATE_STABLE_PKG=1 - fi - - # Proceed and update the main release if the version number has changed - - | - if [ -n "$UPDATE_STABLE_PKG" ]; then - echo "--- Updating main package" - mkdir -p "$APT_ROOT/pool/$DEB_VERSION/main" - cp "$GIT_DEB" "$APT_ROOT/pool/$DEB_VERSION/main/$${PKG_NAME}_$${VERSION}-1_all.deb" - fi + - . .drone/update-deb-packages.sh ### ### Update the Debian (oldstable) packages @@ -556,78 +218,7 @@ steps: - update-components-cache commands: - - | - [ -f .skipci ] && exit 0 - - - echo "-- Copying source directory" - - mkdir -p "$WORKDIR" - - export SRCDIR="$WORKDIR/src" - - cp -r "$PWD" "$SRCDIR" - - cd "$SRCDIR" - - - echo "-- Installing dependencies" - - apt update - - apt install -y curl dpkg-dev gpg git python3 python3-pip python3-setuptools - - - echo "--- Parsing metadata" - - git config --global --add safe.directory "$PWD" - - git pull --rebase origin master --tags - - export VERSION=$(python3 setup.py --version) - - export GIT_VERSION="$VERSION-$(git log --pretty=oneline HEAD...v$VERSION | wc -l)" - - export GIT_BUILD_DIR="$WORKDIR/$${PKG_NAME}_$${GIT_VERSION}_all" - - export GIT_DEB="$WORKDIR/$${PKG_NAME}_$${GIT_VERSION}_all.deb" - - export POOL_PATH="$APT_ROOT/pool/$DEB_VERSION/dev" - - - echo "--- Building git package" - - pip install --prefix="$GIT_BUILD_DIR/usr" --no-cache --no-deps . - - | - find "$GIT_BUILD_DIR" -name "site-packages" | while read dir; do - base="$(dirname "$dir")" - mv "$dir" "$base/dist-packages" - done - - - mkdir -p "$GIT_BUILD_DIR/DEBIAN" - - | - cat < "$GIT_BUILD_DIR/DEBIAN/control" - Package: $PKG_NAME - Version: $GIT_VERSION - Maintainer: Fabio Manganiello - Depends: $(cat platypush/install/requirements/debian.txt | tr '\n' ',' | sed -re 's/,$//' -e 's/,/, /g') - Architecture: all - Homepage: https://platypush.tech - Description: Universal command executor and automation hub. - EOF - - mkdir -p "$POOL_PATH" - - rm -f "$POOL_PATH/"*.deb - - dpkg --build "$GIT_BUILD_DIR" - - - echo "--- Copying $GIT_DEB to $POOL_PATH" - - cp "$GIT_DEB" "$POOL_PATH" - - # If main/all/Packages doesn't exist, then we should create the first main release - - | - [ $(ls "$APT_ROOT/pool/$DEB_VERSION/main/$${PKG_NAME}_$${VERSION}-"*"_all.deb" 2>/dev/null | wc -l) -eq 0 ] && export UPDATE_STABLE_PKG=1 - - - export PKGURL="https://apt.platypush.tech/dists/$DEB_VERSION/main/all/Packages" - - | - [ -z "$UPDATE_STABLE_PKG" ] && - curl -ILs -o /dev/null -w "%{http_code}" "$PKGURL" | - grep -e '^4' >/dev/null && export UPDATE_STABLE_PKG=1 - - # If the published release version differs from the current one, then we should publish a new main release - - | - if [ -z "$UPDATE_STABLE_PKG" ]; then - RELEASED_VERSION=$(curl -s "$PKGURL" | grep -e '^Version: ' | head -1 | awk '{print $2}' | cut -d- -f 1) - [ "$RELEASED_VERSION" != "$VERSION" ] && export UPDATE_STABLE_PKG=1 - fi - - # Proceed and update the main release if the version number has changed - - | - if [ -n "$UPDATE_STABLE_PKG" ]; then - echo "--- Updating main package" - mkdir -p "$APT_ROOT/pool/$DEB_VERSION/main" - cp "$GIT_DEB" "$APT_ROOT/pool/$DEB_VERSION/main/$${PKG_NAME}_$${VERSION}-1_all.deb" - fi + - . .drone/update-deb-packages.sh ### ### Updates the APT repository after new packages have been pushed @@ -658,183 +249,7 @@ steps: - update-debian-oldstable-packages commands: - - | - [ -f .skipci ] && exit 0 - - - echo "-- Installing dependencies" - - apt update - - apt install -y dpkg-dev gpg - - - echo "-- Creating a new apt root folder" - - export TMP_APT_ROOT="/tmp/apt" - - cp -r "$APT_ROOT" "$TMP_APT_ROOT" - - - echo "-- Cleaning up older apt releases" - - | - find "$TMP_APT_ROOT/pool" -mindepth 2 -maxdepth 2 -type d | while read reldir; do - pkg_to_remove=$(( $(ls "$reldir"/*.deb | wc -l) - 1 )) - [ $pkg_to_remove -le 0 ] && continue - ls "$reldir"/*.deb | sort -V | head -n$pkg_to_remove | xargs rm -f - done - - - echo "-- Updating Packages files" - - | - echo "stable\noldstable" | while read distro; do - echo "main\ndev" | while read branch; do - branch_dir="$TMP_APT_ROOT/pool/$distro/$branch" - echo "Checking pool folder: $branch_dir" - [ -d "$branch_dir" ] || mkdir -p "$branch_dir" - dist_dir="$TMP_APT_ROOT/dists/$distro/$branch/all" - mkdir -p "$dist_dir" - pkg_file="$dist_dir/Packages" - dpkg-scanpackages --arch all "$branch_dir" > "$pkg_file" - sed -i "$pkg_file" -re "s|^Filename: $TMP_APT_ROOT/|Filename: |" - cat "$pkg_file" | gzip -9 > "$pkg_file.gz" - echo "Generated Packages file: $pkg_file" - cat "$pkg_file" - done - done - - - echo "-- Updating Release files" - - | - add_hashes() { - dist_dir=$1 - hash_cmd=$2 - hash_label=$3 - - echo "$hash_label:" - find "$dist_dir" -name 'Packages*' | while read file; do - basename="$(echo "$file" | sed -r -e "s|^$dist_dir/||")" - hash="$($hash_cmd "$file" | cut -d" " -f1)" - size="$(wc -c < $file)" - echo " $hash $size $basename" - echo " $hash $size $(echo $basename | sed -re 's|/all/|/binary-i386/|')" - echo " $hash $size $(echo $basename | sed -re 's|/all/|/binary-amd64/|')" - echo " $hash $size $(echo $basename | sed -re 's|/all/|/binary-armel/|')" - echo " $hash $size $(echo $basename | sed -re 's|/all/|/binary-armhf/|')" - echo " $hash $size $(echo $basename | sed -re 's|/all/|/binary-arm64/|')" - done - } - - echo "stable\noldstable" | while read distro; do - dist_dir="$TMP_APT_ROOT/dists/$distro" - components=$(find "$dist_dir" -name Packages | awk -F '/' '{print $(NF-2)}' | uniq | tr '\n' ' ') - release_file="$dist_dir/Release" - - cat < "$release_file" - Origin: Platypush repository - Label: Platypush - Suite: $distro - Codename: $distro - Architectures: i386 amd64 armel armhf arm64 - Components: $components - Description: The official APT repository for Platypush - Date: $(date -Ru) - EOF - - add_hashes "$dist_dir" "md5sum" "MD5Sum" >> "$release_file" - add_hashes "$dist_dir" "sha1sum" "SHA1" >> "$release_file" - add_hashes "$dist_dir" "sha256sum" "SHA256" >> "$release_file" - done - - - echo "-- Generating list files" - - mkdir -p "$TMP_APT_ROOT/lists" - - | - cat < "$TMP_APT_ROOT/lists/platypush-stable-main.list" - deb https://apt.platypush.tech/ stable main - EOF - - - | - cat < "$TMP_APT_ROOT/lists/platypush-stable-dev.list" - deb https://apt.platypush.tech/ stable dev - EOF - - - | - cat < "$TMP_APT_ROOT/lists/platypush-oldstable-main.list" - deb https://apt.platypush.tech/ oldstable main - EOF - - - | - cat < "$TMP_APT_ROOT/lists/platypush-oldstable-dev.list" - deb https://apt.platypush.tech/ oldstable dev - EOF - - - echo "-- Updating index file" - - | - cat < "$TMP_APT_ROOT/index.txt" - Welcome to the Platypush APT repository! - - Project homepage: https://platypush.tech - Source code: https://git.platypush.tech/platypush/platypush - Documentation / API reference: https://docs.platypush.tech - - You can use this APT repository to install Platypush on Debian, Ubuntu or any - Debian-based distro. - - Steps: - - 1. Add this repository's PGP key to your apt keyring - ==================================================== - - $ sudo wget -q -O \\\ - /etc/apt/trusted.gpg.d/platypush.asc \\\ - https://apt.platypush.tech/pubkey.txt - - 2. Add the repository to your sources - ===================================== - - $ sudo wget -q -O \\\ - /etc/apt/sources.list.d/platypush.list \\\ - https://apt.platypush.tech/lists/platypush--.list - - Where: - - - deb_version can be either *stable* (for the current Debian stable version) or - *oldstable* (for the previous Debian stable version) - - branch can be either *main* (for the latest releases) or *dev* (for a package - that is always in sync with the git version) - - For example, to install the latest stable tags on Debian stable: - - $ sudo wget -q -O \\\ - /etc/apt/sources.list.d/platypush.list \\\ - https://apt.platypush.tech/lists/platypush-stable-main.list - - 3. Update your repos - ==================== - - $ sudo apt update - - 4. Install Platypush - ==================== - - $ sudo apt install platypush - EOF - - - echo "-- Importing and refreshing PGP key" - - echo "$PGP_PUBKEY" > "$TMP_APT_ROOT/pubkey.txt" - - export PGP_KEYID=$(echo "$PGP_PUBKEY" | gpg --with-colons --import-options show-only --import --fingerprint | grep -e '^fpr:' | head -1 | awk -F ':' '{print $(NF - 1)}') - - | - cat < "$file.gpg" - cat "$file" | gpg -q --default-key "$PGP_KEYID" -abs --clearsign > "$dirname/InRelease" - done - - - echo "-- Updating the apt repo root" - - export OLD_APT_ROOT="$REPOS_ROOT/oldapt" - - rm -rf "$OLD_APT_ROOT" - - mv "$APT_ROOT" "$OLD_APT_ROOT" - - mv "$TMP_APT_ROOT" "$APT_ROOT" - - - chmod -R a+r "$APT_ROOT" - - chmod a+x "$APT_ROOT" + - . .drone/update-apt-repo.sh ### ### Update the RPM (stable) packages @@ -867,212 +282,7 @@ steps: - update-components-cache commands: - - | - [ -f .skipci ] && exit 0 - - - echo "-- Installing dependencies" - - yum install -y createrepo rpm-build rpm-sign gpg wget yum-utils git python python-pip python-setuptools - - - echo "-- Copying source directory" - - mkdir -p "$WORKDIR" - - export SRCDIR="$WORKDIR/src" - - cp -r "$PWD" "$SRCDIR" - - cd "$SRCDIR" - - mkdir -p "$RPM_ROOT" - - - echo "--- Parsing metadata" - - git config --global --add safe.directory $PWD - - git pull --rebase origin master --tags - - export VERSION=$(python3 setup.py --version) - - export RELNUM="$(git log --pretty=oneline HEAD...v$VERSION | wc -l)" - - export SPECFILE="$WORKDIR/$PKG_NAME.spec" - - export BUILD_DIR="$WORKDIR/build" - - export TMP_RPM_ROOT="$WORKDIR/repo" - - export SRC_URL="https://git.platypush.tech/platypush/platypush/archive/master.tar.gz" - - - echo "--- Creating git package spec" - - | - cat < $SPECFILE - Summary: Universal command executor and automation hub. - Name: $PKG_NAME-git - Version: $VERSION - Release: $RELNUM - URL: https://platypush.tech - Group: System - License: MIT - Packager: Fabio Manganiello - Source: $SRC_URL - Requires: $(cat platypush/install/requirements/fedora.txt | tr '\n' ' ') - Conflicts: $PKG_NAME - Prefix: %{_prefix} - BuildRoot: %{_tmppath}/%{name}-root - - %description - Universal command executor and automation hub. - - %install - mkdir -p %{buildroot}/ - cp -r "$BUILD_DIR"/* %{buildroot}/ - - %clean - - %files - /usr/bin/* - /usr/lib/python$(python3 --version | awk '{print $2}' | cut -d. -f 1,2)/site-packages/platypush - /usr/lib/python$(python3 --version | awk '{print $2}' | cut -d. -f 1,2)/site-packages/platypush-$VERSION.dist-info - - %changelog - * $(date +'%a %b %d %Y') admin - - [Automatic] Release $VERSION-$RELNUM - EOF - - - echo "--- Building git package" - - mkdir -p "$BUILD_DIR" - - pip install --prefix="$BUILD_DIR/usr" --no-cache --no-deps . - - rpmbuild --target "noarch" -bb "$SPECFILE" - - - echo "--- Copying the new RPM package" - - mkdir -p "$TMP_RPM_ROOT" - - cp "$HOME/rpmbuild/RPMS/noarch/$PKG_NAME-git-$VERSION-$RELNUM.noarch.rpm" "$TMP_RPM_ROOT" - - - echo "--- Checking the latest released stable version" - - export LATEST_STABLE_PKG=$(ls -rt "$RPM_ROOT/$PKG_NAME"*.rpm 2>/dev/null | grep -v "$PKG_NAME-git" | tail -1) - - | - if [ -z "$LATEST_STABLE_PKG" ]; then - # If not stable release is available, then create one - export UPDATE_STABLE_PKG=1 - else - # Otherwise, create a new release if the reported version on the repo is different - # from the latest released version. - export LATEST_STABLE_VERSION=$(basename $LATEST_STABLE_PKG | cut -d- -f 2) - if [ "$VERSION" != "$LATEST_STABLE_VERSION" ]; then - export UPDATE_STABLE_PKG=1 - else - # If the version has remained the same, then simply copy the existing RPM to the - # new repository directory. - echo "Copying the existing release $LATEST_STABLE_VERSION to the new repository" - cp "$LATEST_STABLE_PKG" "$TMP_RPM_ROOT" - fi - fi - - # If a new stable release is required, build another RPM - - | - if [ -n "$UPDATE_STABLE_PKG" ]; then - export RELNUM=1 - export SRC_URL="https://git.platypush.tech/platypush/platypush/archive/v$VERSION.tar.gz" - - cat < $SPECFILE - Summary: Universal command executor and automation hub. - Name: $PKG_NAME - Version: $VERSION - Release: $RELNUM - URL: https://platypush.tech - Group: System - License: MIT - Packager: Fabio Manganiello - Source: $SRC_URL - Requires: $(cat platypush/install/requirements/fedora.txt | tr '\n' ' ') - Conflicts: $PKG_NAME-git - Prefix: %{_prefix} - BuildRoot: %{_tmppath}/%{name}-root - - %description - Universal command executor and automation hub. - - %install - mkdir -p %{buildroot}/ - cp -r "$BUILD_DIR"/* %{buildroot}/ - - %clean - - %files - /usr/bin/* - /usr/lib/python$(python3 --version | awk '{print $2}' | cut -d. -f 1,2)/site-packages/platypush - /usr/lib/python$(python3 --version | awk '{print $2}' | cut -d. -f 1,2)/site-packages/platypush-$VERSION.dist-info - - %changelog - * $(date +'%a %b %d %Y') admin - - [Automatic] Release $VERSION-$RELNUM - EOF - - echo "--- Building package for stable release $VERSION" - rpmbuild --target "noarch" -bb "$SPECFILE" - cp "$HOME/rpmbuild/RPMS/noarch/$PKG_NAME-$VERSION-$RELNUM.noarch.rpm" "$TMP_RPM_ROOT" - fi - - - echo "--- Importing the repository keys" - - | - cat < $HOME/.rpmmacros - %signature gpg - %_gpg_name $PGP_KEYID - EOF - - - echo "--- Signing the new RPM packages" - - rpm --addsign "$TMP_RPM_ROOT"/*.rpm - - - echo "--- Creating a new copy of the RPM repository" - - createrepo "$TMP_RPM_ROOT" - - gpg --detach-sign --armor "$TMP_RPM_ROOT/repodata/repomd.xml" - - - | - cat < "$TMP_RPM_ROOT/platypush.repo" - [platypush] - name=Platypush repository - baseurl=https://rpm.platypush.tech - enabled=1 - type=rpm - gpgcheck=1 - gpgkey=https://rpm.platypush.tech/pubkey.txt - EOF - - - | - cat < "$TMP_RPM_ROOT/index.txt" - Welcome to the Platypush RPM repository! - - Project homepage: https://platypush.tech - Source code: https://git.platypush.tech/platypush/platypush - Documentation / API reference: https://docs.platypush.tech - - You can use this RPM repository to install Platypush on Fedora or other - RPM-based distros - as long as they are compatible with the latest Fedora - release. - - Steps: - - 1. Add the repository to your sources - ===================================== - - $ sudo yum config-manager --add-repo https://rpm.platypush.tech/platypush.repo - - 2. Install Platypush - ==================== - - $ sudo yum install platypush - - Or, if you want to install a version always up-to-date with the git repo: - - $ sudo yum install platypush-git - EOF - - - | - cat < "$TMP_RPM_ROOT/pubkey.txt" - $PGP_PUBKEY - EOF - - - echo "--- Updating the repository" - - export NEW_RPM_ROOT="$REPOS_ROOT/rpm_new" - - export OLD_RPM_ROOT="$REPOS_ROOT/rpm_old" - - cp -r "$TMP_RPM_ROOT" "$NEW_RPM_ROOT" - - rm -rf "$TMP_RPM_ROOT" - - mv "$RPM_ROOT" "$OLD_RPM_ROOT" - - mv "$NEW_RPM_ROOT" "$RPM_ROOT" - - rm -rf "$OLD_RPM_ROOT" + - . .drone/update-rpm-repo.sh ### ### Updates the pip package upon new releases @@ -1095,9 +305,7 @@ steps: - docs commands: - - apk add --update --no-cache py3-twine - - python setup.py sdist bdist_wheel - - twine upload dist/platypush-$(python setup.py --version).tar.gz + - . .drone/update-pip-package.sh ### ### Checkpoint step that waits for all the package update diff --git a/.drone/build-ui.sh b/.drone/build-ui.sh new file mode 100755 index 000000000..a9e0a38a2 --- /dev/null +++ b/.drone/build-ui.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +export SKIPCI="$PWD/.skipci" +rm -rf "$SKIPCI" + +. .drone/macros/configure-git.sh + +cd platypush/backend/http/webapp +if [ $(git log --pretty=oneline $DRONE_COMMIT_AFTER...$DRONE_COMMIT_BEFORE . | wc -l) -eq 0 ]; then + echo "No UI changes detected, skipping build" + exit 0 +fi + + +if [ "$(git log --pretty=format:%s HEAD...HEAD~1 | head -1)" == "[Automatic] Updated UI files" ]; then + echo "UI changes have already been committed, skipping build" + exit 0 +fi + +rm -rf dist node_modules +npm install +npm run build + +if [ $(git status --porcelain dist | wc -l) -eq 0 ]; then + echo "No build files have been changed" + exit 0 +fi + +# Create a .skipci file to mark the fact that the next steps should be skipped +# (we're going to do another push anyway, so another pipeline will be triggered) +touch "$SKIPCI" + +. .drone/macros/configure-ssh.sh +. .drone/macros/configure-gpg.sh + +git add dist +git commit dist -S -m "[Automatic] Updated UI files" --no-verify +git remote rm origin +git remote add origin git@git.platypush.tech:platypush/platypush.git +git push -f origin master + +# Restore the original git configuration +mv "$TMP_GIT_CONF" "$GIT_CONF" diff --git a/.drone/github-mirror.sh b/.drone/github-mirror.sh new file mode 100755 index 000000000..70bba95e3 --- /dev/null +++ b/.drone/github-mirror.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +. .drone/macros/configure-git.sh +. .drone/macros/configure-ssh.sh + +ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null + +# Clone the repository +git remote add github git@github.com:/BlackLight/platypush.git +git pull --rebase github "$(git branch | head -1 | awk '{print $2}')" || echo "No such branch on Github" + +# Push the changes to the GitHub mirror +git push --all -v github diff --git a/.drone/macros/configure-git.sh b/.drone/macros/configure-git.sh new file mode 100755 index 000000000..417356384 --- /dev/null +++ b/.drone/macros/configure-git.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +# Install git +if [ -z "$(which git)" ]; then + if [ -n "$(which apt-get)" ]; then + apt-get update + apt-get install -y git + elif [ -n "$(which apk)" ]; then + apk add --update --no-cache git + elif [ -n "$(which yum)" ]; then + yum install -y git + elif [ -n "$(which dnf)" ]; then + dnf install -y git + elif [ -n "$(which pacman)" ]; then + pacman -Sy --noconfirm git + else + echo "Could not find a package manager to install git" + exit 1 + fi +fi + +# Backup the original git configuration before changing attributes +export GIT_CONF="$PWD/.git/config" +export TMP_GIT_CONF=/tmp/git.config.orig +cp "$GIT_CONF" "$TMP_GIT_CONF" + +git config --global --add safe.directory "$PWD" +git config user.name "Platypush CI/CD Automation" +git config user.email "admin@platypush.tech" diff --git a/.drone/macros/configure-gpg.sh b/.drone/macros/configure-gpg.sh new file mode 100755 index 000000000..28da1b998 --- /dev/null +++ b/.drone/macros/configure-gpg.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +[ -z "$PGP_KEY" ] && echo "PGP_KEY is not set" && exit 1 +[ -z "$PGP_KEY_ID" ] && echo "PGP_KEY_ID is not set" && exit 1 + +# Install gpg +if [ -z "$(which gpg)" ]; then + if [ -n "$(which apt-get)" ]; then + apt-get update + apt-get install -y gnupg + elif [ -n "$(which apk)" ]; then + apk add --update --no-cache bash gnupg + elif [ -n "$(which yum)" ]; then + yum install -y gnupg + elif [ -n "$(which dnf)" ]; then + dnf install -y gnupg + elif [ -n "$(which pacman)" ]; then + pacman -Sy --noconfirm gnupg + else + echo "Could not find a package manager to install gnupg" + exit 1 + fi +fi + +cat < ~/.ssh/id_rsa.pub + +cat < ~/.ssh/id_rsa +$SSH_PRIVKEY +EOF + +chmod 0600 ~/.ssh/id_rsa +ssh-keyscan git.platypush.tech >> ~/.ssh/known_hosts 2>/dev/null diff --git a/.drone/rebuild-docs.sh b/.drone/rebuild-docs.sh new file mode 100755 index 000000000..84915b7f2 --- /dev/null +++ b/.drone/rebuild-docs.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +echo "Installing required build dependencies" +apk add --update --no-cache git make py3-sphinx py3-myst-parser py3-pip $(cat platypush/install/requirements/alpine.txt) +pip install -U sphinx-rtd-theme sphinx-book-theme --break-system-packages +pip install . --break-system-packages +mkdir -p /docs/current +export APPDIR="$PWD" +rm -rf "$APPDIR/docs/build" + +echo "Building the updated documentation" +cd "$APPDIR/docs/source" +git clone 'https://git.platypush.tech/platypush/platypush.wiki.git' wiki + +echo "Linking the wiki to the Sphinx index" +cd wiki +ln -s Home.md index.md +cd "$APPDIR/docs" +make html +rm -f config*.yaml +cd "$APPDIR" + +echo "Copying the new documentation files to the target folder" +mv -v "$APPDIR/docs/build" /docs/new +cd /docs +mv current old +mv new current +rm -rf old diff --git a/.drone/run-tests.sh b/.drone/run-tests.sh new file mode 100755 index 000000000..74450a671 --- /dev/null +++ b/.drone/run-tests.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +apk add --update --no-cache $(cat platypush/install/requirements/alpine.txt) +pip install . --break-system-packages +pip install -r requirements-tests.txt --break-system-packages +pytest tests diff --git a/.drone/sync-stable-branch.sh b/.drone/sync-stable-branch.sh new file mode 100755 index 000000000..156afe54a --- /dev/null +++ b/.drone/sync-stable-branch.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +. .drone/macros/configure-git.sh +. .drone/macros/configure-ssh.sh +. .drone/macros/configure-gpg.sh + +# Git configuration +git remote rm origin +git remote add origin git@git.platypush.tech:platypush/platypush.git + +# Merge and push to the `stable` branch +git checkout stable +git rebase master +git push -u origin stable +git checkout master + +# Restore the original git configuration +mv "$TMP_GIT_CONF" "$GIT_CONF" diff --git a/.drone/update-apt-repo.sh b/.drone/update-apt-repo.sh new file mode 100755 index 000000000..5cc849cdd --- /dev/null +++ b/.drone/update-apt-repo.sh @@ -0,0 +1,166 @@ +#!/bin/sh + + +[ -f .skipci ] && exit 0 + +echo "-- Installing dependencies" +apt update +apt install -y dpkg-dev gpg + +echo "-- Creating a new apt root folder" +export TMP_APT_ROOT="/tmp/apt" +cp -r "$APT_ROOT" "$TMP_APT_ROOT" + +echo "-- Cleaning up older apt releases" + +find "$TMP_APT_ROOT/pool" -mindepth 2 -maxdepth 2 -type d | while read reldir; do + pkg_to_remove=$(( $(ls "$reldir"/*.deb | wc -l) - 1 )) + [ $pkg_to_remove -le 0 ] && continue + ls "$reldir"/*.deb | sort -V | head -n$pkg_to_remove | xargs rm -f +done + +echo "-- Updating Packages files" + +echo "stable\noldstable" | while read distro; do + echo "main\ndev" | while read branch; do + branch_dir="$TMP_APT_ROOT/pool/$distro/$branch" + echo "Checking pool folder: $branch_dir" + [ -d "$branch_dir" ] || mkdir -p "$branch_dir" + dist_dir="$TMP_APT_ROOT/dists/$distro/$branch/all" + mkdir -p "$dist_dir" + pkg_file="$dist_dir/Packages" + dpkg-scanpackages --arch all "$branch_dir" > "$pkg_file" + sed -i "$pkg_file" -re "s|^Filename: $TMP_APT_ROOT/|Filename: |" + cat "$pkg_file" | gzip -9 > "$pkg_file.gz" + echo "Generated Packages file: $pkg_file" + cat "$pkg_file" + done +done + +echo "-- Updating Release files" + +add_hashes() { + dist_dir=$1 + hash_cmd=$2 + hash_label=$3 + + echo "$hash_label:" + find "$dist_dir" -name 'Packages*' | while read file; do + basename="$(echo "$file" | sed -r -e "s|^$dist_dir/||")" + hash="$($hash_cmd "$file" | cut -d" " -f1)" + size="$(wc -c < $file)" + echo " $hash $size $basename" + echo " $hash $size $(echo $basename | sed -re 's|/all/|/binary-i386/|')" + echo " $hash $size $(echo $basename | sed -re 's|/all/|/binary-amd64/|')" + echo " $hash $size $(echo $basename | sed -re 's|/all/|/binary-armel/|')" + echo " $hash $size $(echo $basename | sed -re 's|/all/|/binary-armhf/|')" + echo " $hash $size $(echo $basename | sed -re 's|/all/|/binary-arm64/|')" + done +} + +echo "stable\noldstable" | while read distro; do + dist_dir="$TMP_APT_ROOT/dists/$distro" + components=$(find "$dist_dir" -name Packages | awk -F '/' '{print $(NF-2)}' | uniq | tr '\n' ' ') + release_file="$dist_dir/Release" + + cat < "$release_file" +Origin: Platypush repository +Label: Platypush +Suite: $distro +Codename: $distro +Architectures: i386 amd64 armel armhf arm64 +Components: $components +Description: The official APT repository for Platypush +Date: $(date -Ru) +EOF + + add_hashes "$dist_dir" "md5sum" "MD5Sum" >> "$release_file" + add_hashes "$dist_dir" "sha1sum" "SHA1" >> "$release_file" + add_hashes "$dist_dir" "sha256sum" "SHA256" >> "$release_file" +done + +echo "-- Generating list files" +mkdir -p "$TMP_APT_ROOT/lists" + +for distro in stable oldstable; do + for branch in main dev; do + echo "deb https://apt.platypush.tech/ $distro $branch" > "$TMP_APT_ROOT/lists/platypush-$distro-$branch.list" + done +done + +echo "-- Updating index file" + +cat < "$TMP_APT_ROOT/index.txt" +Welcome to the Platypush APT repository! + +Project homepage: https://platypush.tech +Source code: https://git.platypush.tech/platypush/platypush +Documentation / API reference: https://docs.platypush.tech + +You can use this APT repository to install Platypush on Debian, Ubuntu or any +Debian-based distro. + +Steps: + +1. Add this repository's PGP key to your apt keyring +==================================================== + +$ sudo wget -q -O \\\ + /etc/apt/trusted.gpg.d/platypush.asc \\\ + https://apt.platypush.tech/pubkey.txt + +2. Add the repository to your sources +===================================== + +$ sudo wget -q -O \\\ + /etc/apt/sources.list.d/platypush.list \\\ + https://apt.platypush.tech/lists/platypush--.list + +Where: + +- deb_version can be either *stable* (for the current Debian stable version) or + *oldstable* (for the previous Debian stable version) +- branch can be either *main* (for the latest releases) or *dev* (for a package + that is always in sync with the git version) + +For example, to install the latest stable tags on Debian stable: + +$ sudo wget -q -O \\\ + /etc/apt/sources.list.d/platypush.list \\\ + https://apt.platypush.tech/lists/platypush-stable-main.list + +3. Update your repos +==================== + +$ sudo apt update + +4. Install Platypush +==================== + +$ sudo apt install platypush +EOF + +echo "-- Importing and refreshing PGP key" +echo "$PGP_PUBKEY" > "$TMP_APT_ROOT/pubkey.txt" +export PGP_KEYID=$(echo "$PGP_PUBKEY" | gpg --with-colons --import-options show-only --import --fingerprint | grep -e '^fpr:' | head -1 | awk -F ':' '{print $(NF - 1)}') + +cat < "$file.gpg" + cat "$file" | gpg -q --default-key "$PGP_KEYID" -abs --clearsign > "$dirname/InRelease" +done + +echo "-- Updating the apt repo root" +export OLD_APT_ROOT="$REPOS_ROOT/oldapt" +rm -rf "$OLD_APT_ROOT" +mv "$APT_ROOT" "$OLD_APT_ROOT" +mv "$TMP_APT_ROOT" "$APT_ROOT" + +chmod -R a+r "$APT_ROOT" +chmod a+x "$APT_ROOT" diff --git a/.drone/update-arch-packages.sh b/.drone/update-arch-packages.sh new file mode 100755 index 000000000..695cdf580 --- /dev/null +++ b/.drone/update-arch-packages.sh @@ -0,0 +1,67 @@ +#!/bin/sh + +[ -f .skipci ] && exit 0 + +apk add --update --no-cache curl pacman sudo + +. .drone/macros/configure-ssh.sh +. .drone/macros/configure-git.sh + +git pull --rebase origin master --tags + +export VERSION=$(python setup.py --version) +export HEAD=$(git log --pretty=format:%h HEAD...HEAD~1 | head -1) +export GIT_VERSION="$VERSION.r$(git log --pretty=oneline HEAD...v$VERSION | wc -l).g${HEAD}" +export TAG_URL="https://git.platypush.tech/platypush/platypush/archive/v$VERSION.tar.gz" + +ssh-keyscan aur.archlinux.org >> ~/.ssh/known_hosts 2>/dev/null +adduser -u 1000 -D build +mkdir -p "$WORKDIR" + +echo "--- Updating Arch git version" +export PKGDIR=$WORKDIR/git +git clone ssh://aur@aur.archlinux.org/platypush-git.git "$PKGDIR" +git config --global --add safe.directory "$PKGDIR" +chown -R build "$PKGDIR" +cd "$PKGDIR" + +sed -i 'PKGBUILD' -r \ + -e "s/^pkgver=.*/pkgver=$GIT_VERSION/" \ + -e "s/^pkgrel=.*/pkgrel=1/" \ + +sudo -u build makepkg --printsrcinfo > .SRCINFO +export FILES_CHANGED=$(git status --porcelain --untracked-files=no | wc -l) + +if [ $FILES_CHANGED -gt 0 ]; then + echo "--- Pushing git package version $GIT_VERSION" + git commit -a -m '[Automatic] Package update' + git push origin master +fi + +echo "--- Updating Arch stable version" +export PKGDIR="$WORKDIR/stable" +git clone ssh://aur@aur.archlinux.org/platypush.git "$PKGDIR" +git config --global --add safe.directory "$PKGDIR" +chown -R build "$PKGDIR" +cd "$PKGDIR" +export RELEASED_VERSION=$(grep -e '^pkgver=' PKGBUILD | sed -r -e 's/^pkgver=(.*)\s*/\1/') + +if [ "$RELEASED_VERSION" == "$VERSION" ]; then + exit 0 +fi + +export TAG_CHECKSUM=$(curl --silent "$TAG_URL" | sha512sum | awk '{print $1}') + +sed -i 'PKGBUILD' -r \ + -e "s/^pkgver=.*/pkgver=$VERSION/" \ + -e "s/^pkgrel=.*/pkgrel=1/" \ + -e "s/^sha512sums=.*/sha512sums=('$TAG_CHECKSUM')/" + +sudo -u build makepkg --printsrcinfo > .SRCINFO +export FILES_CHANGED=$(git status --porcelain --untracked-files=no | wc -l) + +if [ $FILES_CHANGED -gt 0 ]; then + echo "--- Pushing stable package version $VERSION" + git commit -a -m '[Automatic] Package update' + git push origin master +fi diff --git a/.drone/update-components-cache.sh b/.drone/update-components-cache.sh new file mode 100755 index 000000000..cc9a5b614 --- /dev/null +++ b/.drone/update-components-cache.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +export SKIPCI="$PWD/.skipci" +export CACHEFILE="$PWD/platypush/components.json.gz" + +[ -f "$SKIPCI" ] && exit 0 + +# Backup the original git configuration before changing attributes +export GIT_CONF="$PWD/.git/config" +export TMP_GIT_CONF="/tmp/git.config.orig" +cp "$GIT_CONF" "$TMP_GIT_CONF" + +. .drone/macros/configure-git.sh + +# Only regenerate the components cache if either the plugins, backends, +# events or schemas folders have some changes (excluding the webapp files). +if [ -z "$(git log --pretty=oneline $DRONE_COMMIT_AFTER...$DRONE_COMMIT_BEFORE -- platypush/backend platypush/plugins platypush/schemas platypush/message/event ':(exclude)platypush/backend/http/webapp')" ]; then + echo 'No changes to the components file' + exit 0 +fi + +. .drone/macros/configure-ssh.sh +. .drone/macros/configure-gpg.sh + +echo 'Updating components cache' +apk add --update --no-cache $(cat platypush/install/requirements/alpine.txt) +pip install . --break-system-packages + +python - < "$GIT_BUILD_DIR/DEBIAN/control" +Package: $PKG_NAME +Version: $GIT_VERSION +Maintainer: Fabio Manganiello +Depends: $(cat platypush/install/requirements/debian.txt | tr '\n' ',' | sed -re 's/,$//' -e 's/,/, /g') +Architecture: all +Homepage: https://platypush.tech +Description: Universal command executor and automation hub. +EOF + +mkdir -p "$POOL_PATH" +rm -f "$POOL_PATH/"*.deb +dpkg --build "$GIT_BUILD_DIR" + +echo "--- Copying $GIT_DEB to $POOL_PATH" +cp "$GIT_DEB" "$POOL_PATH" + +# If main/all/Packages doesn't exist, then we should create the first main release +[ $(ls "$APT_ROOT/pool/$DEB_VERSION/main/${PKG_NAME}_${VERSION}-"*"_all.deb" 2>/dev/null | wc -l) -eq 0 ] && export UPDATE_STABLE_PKG=1 + +export PKGURL="https://apt.platypush.tech/dists/$DEB_VERSION/main/all/Packages" + +[ -z "$UPDATE_STABLE_PKG" ] && + curl -ILs -o /dev/null -w "%{http_code}" "$PKGURL" | + grep -e '^4' >/dev/null && export UPDATE_STABLE_PKG=1 + +# If the published release version differs from the current one, then we should publish a new main release +if [ -z "$UPDATE_STABLE_PKG" ]; then + RELEASED_VERSION=$(curl -s "$PKGURL" | grep -e '^Version: ' | head -1 | awk '{print $2}' | cut -d- -f 1) + [ "$RELEASED_VERSION" != "$VERSION" ] && export UPDATE_STABLE_PKG=1 +fi + +# Proceed and update the main release if the version number has changed +if [ -n "$UPDATE_STABLE_PKG" ]; then + echo "--- Updating main package" + mkdir -p "$APT_ROOT/pool/$DEB_VERSION/main" + cp "$GIT_DEB" "$APT_ROOT/pool/$DEB_VERSION/main/${PKG_NAME}_${VERSION}-1_all.deb" +fi diff --git a/.drone/update-pip-package.sh b/.drone/update-pip-package.sh new file mode 100755 index 000000000..5bd6e25ad --- /dev/null +++ b/.drone/update-pip-package.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +apk add --update --no-cache py3-twine +python setup.py sdist bdist_wheel +twine upload dist/platypush-$(python setup.py --version).tar.gz diff --git a/.drone/update-rpm-repo.sh b/.drone/update-rpm-repo.sh new file mode 100755 index 000000000..37422eb4d --- /dev/null +++ b/.drone/update-rpm-repo.sh @@ -0,0 +1,201 @@ +#!/bin/sh + +[ -f .skipci ] && exit 0 + +echo "-- Installing dependencies" +yum install -y createrepo rpm-build rpm-sign gpg wget yum-utils git python python-pip python-setuptools + +echo "-- Copying source directory" +mkdir -p "$WORKDIR" +export SRCDIR="$WORKDIR/src" +cp -r "$PWD" "$SRCDIR" +cd "$SRCDIR" +mkdir -p "$RPM_ROOT" + +echo "--- Parsing metadata" +git config --global --add safe.directory $PWD +git pull --rebase origin master --tags +export VERSION=$(python3 setup.py --version) +export RELNUM="$(git log --pretty=oneline HEAD...v$VERSION | wc -l)" +export SPECFILE="$WORKDIR/$PKG_NAME.spec" +export BUILD_DIR="$WORKDIR/build" +export TMP_RPM_ROOT="$WORKDIR/repo" +export SRC_URL="https://git.platypush.tech/platypush/platypush/archive/master.tar.gz" + +echo "--- Creating git package spec" + +cat < $SPECFILE +Summary: Universal command executor and automation hub. +Name: $PKG_NAME-git +Version: $VERSION +Release: $RELNUM +URL: https://platypush.tech +Group: System +License: MIT +Packager: Fabio Manganiello +Source: $SRC_URL +Requires: $(cat platypush/install/requirements/fedora.txt | tr '\n' ' ') +Conflicts: $PKG_NAME +Prefix: %{_prefix} +BuildRoot: %{_tmppath}/%{name}-root + +%description +Universal command executor and automation hub. + +%install +mkdir -p %{buildroot}/ +cp -r "$BUILD_DIR"/* %{buildroot}/ + +%clean + +%files +/usr/bin/* +/usr/lib/python$(python3 --version | awk '{print $2}' | cut -d. -f 1,2)/site-packages/platypush +/usr/lib/python$(python3 --version | awk '{print $2}' | cut -d. -f 1,2)/site-packages/platypush-$VERSION.dist-info + +%changelog +* $(date +'%a %b %d %Y') admin +- [Automatic] Release $VERSION-$RELNUM +EOF + +echo "--- Building git package" +mkdir -p "$BUILD_DIR" +pip install --prefix="$BUILD_DIR/usr" --no-cache --no-deps . +rpmbuild --target "noarch" -bb "$SPECFILE" + +echo "--- Copying the new RPM package" +mkdir -p "$TMP_RPM_ROOT" +cp "$HOME/rpmbuild/RPMS/noarch/$PKG_NAME-git-$VERSION-$RELNUM.noarch.rpm" "$TMP_RPM_ROOT" + +echo "--- Checking the latest released stable version" +export LATEST_STABLE_PKG=$(ls -rt "$RPM_ROOT/$PKG_NAME"*.rpm 2>/dev/null | grep -v "$PKG_NAME-git" | tail -1) + +if [ -z "$LATEST_STABLE_PKG" ]; then + # If not stable release is available, then create one + export UPDATE_STABLE_PKG=1 +else + # Otherwise, create a new release if the reported version on the repo is different + # from the latest released version. + export LATEST_STABLE_VERSION=$(basename $LATEST_STABLE_PKG | cut -d- -f 2) + if [ "$VERSION" != "$LATEST_STABLE_VERSION" ]; then + export UPDATE_STABLE_PKG=1 + else + # If the version has remained the same, then simply copy the existing RPM to the + # new repository directory. + echo "Copying the existing release $LATEST_STABLE_VERSION to the new repository" + cp "$LATEST_STABLE_PKG" "$TMP_RPM_ROOT" + fi +fi + +# If a new stable release is required, build another RPM +if [ -n "$UPDATE_STABLE_PKG" ]; then + export RELNUM=1 + export SRC_URL="https://git.platypush.tech/platypush/platypush/archive/v$VERSION.tar.gz" + + cat < $SPECFILE +Summary: Universal command executor and automation hub. +Name: $PKG_NAME +Version: $VERSION +Release: $RELNUM +URL: https://platypush.tech +Group: System +License: MIT +Packager: Fabio Manganiello +Source: $SRC_URL +Requires: $(cat platypush/install/requirements/fedora.txt | tr '\n' ' ') +Conflicts: $PKG_NAME-git +Prefix: %{_prefix} +BuildRoot: %{_tmppath}/%{name}-root + +%description +Universal command executor and automation hub. + +%install +mkdir -p %{buildroot}/ +cp -r "$BUILD_DIR"/* %{buildroot}/ + +%clean + +%files +/usr/bin/* +/usr/lib/python$(python3 --version | awk '{print $2}' | cut -d. -f 1,2)/site-packages/platypush +/usr/lib/python$(python3 --version | awk '{print $2}' | cut -d. -f 1,2)/site-packages/platypush-$VERSION.dist-info + +%changelog +* $(date +'%a %b %d %Y') admin +- [Automatic] Release $VERSION-$RELNUM +EOF + + echo "--- Building package for stable release $VERSION" + rpmbuild --target "noarch" -bb "$SPECFILE" + cp "$HOME/rpmbuild/RPMS/noarch/$PKG_NAME-$VERSION-$RELNUM.noarch.rpm" "$TMP_RPM_ROOT" +fi + +echo "--- Importing the repository keys" +cat < $HOME/.rpmmacros +%signature gpg +%_gpg_name $PGP_KEYID +EOF + +echo "--- Signing the new RPM packages" +rpm --addsign "$TMP_RPM_ROOT"/*.rpm + +echo "--- Creating a new copy of the RPM repository" +createrepo "$TMP_RPM_ROOT" +gpg --detach-sign --armor "$TMP_RPM_ROOT/repodata/repomd.xml" + +cat < "$TMP_RPM_ROOT/platypush.repo" +[platypush] +name=Platypush repository +baseurl=https://rpm.platypush.tech +enabled=1 +type=rpm +gpgcheck=1 +gpgkey=https://rpm.platypush.tech/pubkey.txt +EOF + +cat < "$TMP_RPM_ROOT/index.txt" +Welcome to the Platypush RPM repository! + +Project homepage: https://platypush.tech +Source code: https://git.platypush.tech/platypush/platypush +Documentation / API reference: https://docs.platypush.tech + +You can use this RPM repository to install Platypush on Fedora or other +RPM-based distros - as long as they are compatible with the latest Fedora +release. + +Steps: + +1. Add the repository to your sources +===================================== + +$ sudo yum config-manager --add-repo https://rpm.platypush.tech/platypush.repo + +2. Install Platypush +==================== + +$ sudo yum install platypush + +Or, if you want to install a version always up-to-date with the git repo: + +$ sudo yum install platypush-git +EOF + +cat < "$TMP_RPM_ROOT/pubkey.txt" +$PGP_PUBKEY +EOF + +echo "--- Updating the repository" +export NEW_RPM_ROOT="$REPOS_ROOT/rpm_new" +export OLD_RPM_ROOT="$REPOS_ROOT/rpm_old" +cp -r "$TMP_RPM_ROOT" "$NEW_RPM_ROOT" +rm -rf "$TMP_RPM_ROOT" +mv "$RPM_ROOT" "$OLD_RPM_ROOT" +mv "$NEW_RPM_ROOT" "$RPM_ROOT" +rm -rf "$OLD_RPM_ROOT"