From 133f2ad712b9d3c5f79e7c65efb0d6432c5bcd0a Mon Sep 17 00:00:00 2001 From: Paul Spooren Date: Wed, 14 Aug 2019 16:22:49 -1000 Subject: [PATCH] split TARGETS in stacks of 10 ImageBuilder/SDKs are build for each target + master & 19.07-snapshot branch. This is to much for most CI runners, this separates the job and also allows running it in parrallel if more workers are available. Also add PGP key for 19.07 release builds. Introduce $CI_COMMIT_REF_SLUG for testing. Introduce $SKIP_ROOTFS for older builds not supporting docker images. Signed-off-by: Paul Spooren --- .gitlab-ci.yml | 222 ++++++++++++++++++++++++++--------------- docker-build.sh | 20 ++-- docker-common.sh | 4 + docker-imagebuilder.sh | 19 ++-- docker-rootfs.sh | 21 ++-- docker-sdk.sh | 19 ++-- 6 files changed, 182 insertions(+), 123 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8b68637..2248e99 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,8 +1,15 @@ variables: - TERM: xterm - TARGETS: "cns3xxx-generic mvebu-cortexa53 mvebu-cortexa72 mvebu-cortexa9 ipq40xx-generic ipq806x-generic layerscape-armv7 layerscape-armv8_32b layerscape-armv8_64b imx6-generic octeontx-generic sunxi-cortexa8 sunxi-cortexa53 sunxi-cortexa7 bcm53xx-generic brcm47xx-mips74k brcm47xx-generic brcm47xx-legacy ath79-nand ath79-generic ath79-tiny ath25-generic ar7-ac49x ar7-generic kirkwood-generic apm821xx-sata apm821xx-nand ramips-rt305x ramips-rt3883 ramips-mt76x8 ramips-mt7620 ramips-rt288x ramips-mt7621 pistachio-generic gemini-generic brcm2708-bcm2708 brcm2708-bcm2709 brcm2708-bcm2710 x86-geode x86-generic x86-legacy x86-64 lantiq-xway_legacy lantiq-xrx200 lantiq-ase lantiq-falcon lantiq-xway mediatek-mt7622 mediatek-mt7623 mpc85xx-p1020 mpc85xx-p2020 mpc85xx-generic tegra-generic zynq-generic archs38-generic ixp4xx-harddisk ixp4xx-generic mxs-generic oxnas-ox820 octeon-generic armvirt-32 armvirt-64 arc770-generic samsung-s5pv210 omap-generic ar71xx-nand ar71xx-generic ar71xx-tiny ar71xx-mikrotik brcm63xx-generic brcm63xx-smp at91-sama5d3 at91-sama5d4 at91-sam9x at91-sama5 at91-legacy at91-sama5d2 rb532-generic malta-be" + BRANCH: master + TARGETS_1: "cns3xxx-generic mvebu-cortexa53 mvebu-cortexa72 mvebu-cortexa9 ipq40xx-generic ipq806x-generic layerscape-armv7 layerscape-armv8_32b layerscape-armv8_64b imx6-generic" + TARGETS_2: "octeontx-generic sunxi-cortexa8 sunxi-cortexa53 sunxi-cortexa7 bcm53xx-generic brcm47xx-mips74k brcm47xx-generic brcm47xx-legacy ath79-nand ath79-generic" + TARGETS_3: "ath79-tiny ath25-generic ar7-ac49x ar7-generic kirkwood-generic apm821xx-sata apm821xx-nand ramips-rt305x ramips-rt3883 ramips-mt76x8" + TARGETS_4: "ramips-mt7620 ramips-rt288x ramips-mt7621 pistachio-generic gemini-generic brcm2708-bcm2708 brcm2708-bcm2709 brcm2708-bcm2710 x86-geode x86-generic" + TARGETS_5: "x86-legacy x86-64 lantiq-xway_legacy lantiq-xrx200 lantiq-ase lantiq-falcon lantiq-xway mediatek-mt7622 mediatek-mt7623 mpc85xx-p1020" + TARGETS_6: "mpc85xx-p2020 mpc85xx-generic tegra-generic zynq-generic archs38-generic ixp4xx-harddisk ixp4xx-generic mxs-generic oxnas-ox820 octeon-generic" + TARGETS_7: "armvirt-32 armvirt-64 arc770-generic samsung-s5pv210 omap-generic ar71xx-nand ar71xx-generic ar71xx-tiny ar71xx-mikrotik brcm63xx-generic" + TARGETS_8: "brcm63xx-smp at91-sama5d3 at91-sama5d4 at91-sam9x at91-sama5 at91-legacy at91-sama5d2 rb532-generic malta-be" -build-imagebuilder: +.build: image: docker:latest stage: build services: @@ -11,24 +18,8 @@ build-imagebuilder: - apk add curl rsync bash gnupg outils-signify - bash docker-common.sh - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - script: - - bash docker-imagebuilder.sh - - docker tag "$DOCKER_IMAGE:latest" "$DOCKER_IMAGE:imagebuilder" - - docker push "$DOCKER_IMAGE:imagebuilder" - variables: - TARGETS: "x86-64" - BRANCHES: "master 19.07-SNAPSHOT" - DOCKER_IMAGE: "$CI_REGISTRY_IMAGE" -test-imagebuilder: - image: "$CI_REGISTRY_IMAGE:imagebuilder" - stage: test - script: - - cd ~/openwrt - - make image - - ls ./bin/targets/x86/64/*combined-squashfs.img.gz - -deploy-imagebuilder: +.deploy: image: docker:latest stage: deploy only: @@ -39,32 +30,53 @@ deploy-imagebuilder: - apk add curl rsync bash gnupg outils-signify - bash docker-common.sh - docker login -u "$DOCKER_USER" -p "$DOCKER_PASS" + +.deploy-imagebuilder: + extends: .deploy + variables: + DOCKER_IMAGE: "openwrtorg/imagebuilder" script: - bash docker-imagebuilder.sh - docker push "$DOCKER_IMAGE" - variables: - DOCKER_IMAGE: "openwrtorg/imagebuilder" -build-sdk: - image: docker:latest - stage: build - services: - - docker:dind - before_script: - - apk add curl rsync bash gnupg outils-signify - - bash docker-common.sh - - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY +.deploy-sdk: + extends: .deploy + variables: + DOCKER_IMAGE: "openwrtorg/sdk" script: - bash docker-sdk.sh - - docker tag "$DOCKER_IMAGE:latest" "$DOCKER_IMAGE:sdk" - - docker push "$DOCKER_IMAGE:sdk" + - docker push "$DOCKER_IMAGE" + +build-imagebuilder: + extends: .build variables: TARGETS: "x86-64" - BRANCHES: "master 19.07-SNAPSHOT" DOCKER_IMAGE: "$CI_REGISTRY_IMAGE" + script: + - bash docker-imagebuilder.sh + - docker tag "$DOCKER_IMAGE:x86-64-$BRANCH" "$DOCKER_IMAGE:imagebuilder-$BRANCH-$CI_COMMIT_REF_SLUG" + - docker push "$DOCKER_IMAGE:imagebuilder-$BRANCH-$CI_COMMIT_REF_SLUG" + +test-imagebuilder: + image: "$CI_REGISTRY_IMAGE:imagebuilder-$BRANCH-$CI_COMMIT_REF_SLUG" + stage: test + script: + - cd /home/build/openwrt/ + - make image + - ls ./bin/targets/x86/64/*combined-squashfs.img.gz + +build-sdk: + extends: .build + variables: + TARGETS: "x86-64" + DOCKER_IMAGE: "$CI_REGISTRY_IMAGE" + script: + - bash docker-sdk.sh + - docker tag "$DOCKER_IMAGE:x86-64-$BRANCH" "$DOCKER_IMAGE:sdk-$BRANCH-$CI_COMMIT_REF_SLUG" + - docker push "$DOCKER_IMAGE:sdk-$BRANCH-$CI_COMMIT_REF_SLUG" test-sdk: - image: "$CI_REGISTRY_IMAGE:sdk" + image: "$CI_REGISTRY_IMAGE:sdk-$BRANCH-$CI_COMMIT_REF_SLUG" stage: test script: - cd ~/openwrt @@ -74,63 +86,117 @@ test-sdk: - make package/busybox/compile -j$(nproc) - ls ./bin/packages/x86_64/base/busybox* -deploy-sdk: - image: docker:latest - stage: deploy - only: - - master - services: - - docker:dind - before_script: - - apk add curl rsync bash gnupg outils-signify - - bash docker-common.sh - - docker login -u "$DOCKER_USER" -p "$DOCKER_PASS" - script: - - bash docker-sdk.sh - - docker push "$DOCKER_IMAGE" +deploy-imagebuilder-1: + extends: .deploy-imagebuilder variables: - DOCKER_IMAGE: "openwrtorg/sdk" + TARGETS: "$TARGETS_1" + +deploy-imagebuilder-2: + extends: .deploy-imagebuilder + variables: + TARGETS: "$TARGETS_2" + +deploy-imagebuilder-3: + extends: .deploy-imagebuilder + variables: + TARGETS: "$TARGETS_3" + +deploy-imagebuilder-4: + extends: .deploy-imagebuilder + variables: + TARGETS: "$TARGETS_4" + +deploy-imagebuilder-5: + extends: .deploy-imagebuilder + variables: + TARGETS: "$TARGETS_5" + +deploy-imagebuilder-6: + extends: .deploy-imagebuilder + variables: + TARGETS: "$TARGETS_6" + +deploy-imagebuilder-7: + extends: .deploy-imagebuilder + variables: + TARGETS: "$TARGETS_7" + +deploy-imagebuilder-8: + extends: .deploy-imagebuilder + variables: + TARGETS: "$TARGETS_8" + +deploy-sdk-1: + extends: .deploy-sdk + variables: + TARGETS: "$TARGETS_1" + +deploy-sdk-2: + extends: .deploy-sdk + variables: + TARGETS: "$TARGETS_2" + +deploy-sdk-3: + extends: .deploy-sdk + variables: + TARGETS: "$TARGETS_3" + +deploy-sdk-4: + extends: .deploy-sdk + variables: + TARGETS: "$TARGETS_4" + +deploy-sdk-5: + extends: .deploy-sdk + variables: + TARGETS: "$TARGETS_5" + +deploy-sdk-6: + extends: .deploy-sdk + variables: + TARGETS: "$TARGETS_6" + +deploy-sdk-7: + extends: .deploy-sdk + variables: + TARGETS: "$TARGETS_7" + +deploy-sdk-8: + extends: .deploy-sdk + variables: + TARGETS: "$TARGETS_8" build-rootfs: - image: docker:latest - stage: build - services: - - docker:dind - before_script: - - apk add curl rsync bash gnupg outils-signify - - bash docker-common.sh - - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - script: - - bash docker-rootfs.sh - - docker tag "$DOCKER_IMAGE:latest" "$DOCKER_IMAGE:rootfs" - - docker push "$DOCKER_IMAGE:rootfs" + extends: .build + except: + variables: + - $SKIP_ROOTFS variables: TARGETS: "x86-64" - BRANCHES: "master 19.07-SNAPSHOT" DOCKER_IMAGE: "$CI_REGISTRY_IMAGE" + script: + - bash docker-rootfs.sh + - docker tag "$DOCKER_IMAGE:x86-64-$BRANCH" "$DOCKER_IMAGE:rootfs-$BRANCH-$CI_COMMIT_REF_SLUG" + - docker push "$DOCKER_IMAGE:rootfs-$BRANCH-$CI_COMMIT_REF_SLUG" test-rootfs: - image: "$CI_REGISTRY_IMAGE:rootfs" + image: "$CI_REGISTRY_IMAGE:rootfs-$BRANCH-$CI_COMMIT_REF_SLUG" stage: test + except: + variables: + - $SKIP_ROOTFS script: - ls / - ping -c 3 1.1.1.1 deploy-rootfs: - image: docker:latest - stage: deploy - only: - - master - services: - - docker:dind - before_script: - - apk add curl rsync bash gnupg outils-signify - - bash docker-common.sh - - docker login -u "$DOCKER_USER" -p "$DOCKER_PASS" + extends: .deploy + except: + variables: + - $SKIP_ROOTFS + variables: + DOCKER_IMAGE: "openwrtorg/rootfs" + TARGETS: "x86-64 armvirt-32 armvirt-64" script: - bash docker-rootfs.sh - docker push "$DOCKER_IMAGE" - variables: - DOCKER_IMAGE: "openwrtorg/rootfs" - BRANCHES: "master 19.07-SNAPSHOT" - TARGETS: "x86-64 armvirt-32 armvirt-64" diff --git a/docker-build.sh b/docker-build.sh index 135adbd..b87f944 100755 --- a/docker-build.sh +++ b/docker-build.sh @@ -3,19 +3,17 @@ set -ex DOCKERFILE="${DOCKERFILE:-Dockerfile}" +docker build -t "$DOCKER_IMAGE:$TARGET-$BRANCH" -f "$DOCKERFILE" ./build -if [ "$BRANCH" != "master" ]; then - if [ "$TARGET" != "x86-64" ]; then - docker build -t "$DOCKER_IMAGE:$TARGET-$BRANCH" -f "$DOCKERFILE" ./build - else - docker build -t "$DOCKER_IMAGE:$BRANCH" -f "$DOCKERFILE" ./build - fi -else - if [ "$TARGET" != "x86-64" ]; then - docker build -t "$DOCKER_IMAGE:$TARGET" -f "$DOCKERFILE" ./build - else - docker build -t "$DOCKER_IMAGE:latest" -f "$DOCKERFILE" ./build +if [ "$BRANCH" == "master" ]; then + docker tag "$DOCKER_IMAGE:$TARGET-$BRANCH" "$DOCKER_IMAGE:$TARGET" + if [ "$TARGET" == "x86-64" ]; then + docker tag "$DOCKER_IMAGE:$TARGET-$BRANCH" "$DOCKER_IMAGE:latest" fi fi +if [ "$TARGET" == "x86-64" ]; then + docker tag "$DOCKER_IMAGE:$TARGET-$BRANCH" "$DOCKER_IMAGE:$BRANCH" +fi + rm -rf ./build diff --git a/docker-common.sh b/docker-common.sh index e3488a8..ed01315 100755 --- a/docker-common.sh +++ b/docker-common.sh @@ -10,6 +10,10 @@ export GNUPGHOME="gpg" curl 'https://git.openwrt.org/?p=keyring.git;a=blob_plain;f=gpg/626471F1.asc' | gpg --import \ && echo '54CC74307A2C6DC9CE618269CD84BCED626471F1:6:' | gpg --import-ownertrust +# PGP key for 19.07 release builds +curl 'https://git.openwrt.org/?p=keyring.git;a=blob_plain;f=gpg/2074BE7A.asc' | gpg --import \ + && echo 'D9C6901F45C9B86858687DFF28A39BC32074BE7A:6:' | gpg --import-ownertrust + # LEDE Release Builder (17.01 "Reboot" Signing Key) curl 'https://git.openwrt.org/?p=keyring.git;a=blob_plain;f=gpg/D52BBB6B.asc' | gpg --import \ && echo 'B09BE781AE8A0CD4702FDCD3833C6010D52BBB6B:6:' | gpg --import-ownertrust diff --git a/docker-imagebuilder.sh b/docker-imagebuilder.sh index 65b516d..3da4c29 100755 --- a/docker-imagebuilder.sh +++ b/docker-imagebuilder.sh @@ -3,21 +3,18 @@ set -ex TARGETS="${TARGETS:-x86-64}" -BRANCHES="${BRANCHES:-master}" +export BRANCH="${BRANCH:-master}" export DOCKER_IMAGE="${DOCKER_IMAGE:-openwrt-imagebuilder}" export DOWNLOAD_FILE="openwrt-imagebuilder*x86_64.tar.xz" for TARGET in $TARGETS ; do export TARGET - for BRANCH in $BRANCHES; do - export BRANCH - if [ "$BRANCH" == "master" ]; then - export DOWNLOAD_PATH="snapshots/targets/$(echo $TARGET | tr '-' '/')" - else - export DOWNLOAD_PATH="releases/$BRANCH/targets/$(echo $TARGET | tr '-' '/')" - fi + if [ "$BRANCH" == "master" ]; then + export DOWNLOAD_PATH="snapshots/targets/$(echo $TARGET | tr '-' '/')" + else + export DOWNLOAD_PATH="releases/$BRANCH/targets/$(echo $TARGET | tr '-' '/')" + fi - ./docker-download.sh || exit 1 - ./docker-build.sh || exit 1 - done + ./docker-download.sh || continue + ./docker-build.sh || exit 1 done diff --git a/docker-rootfs.sh b/docker-rootfs.sh index 0fe7a7b..60822b5 100755 --- a/docker-rootfs.sh +++ b/docker-rootfs.sh @@ -3,25 +3,22 @@ set -ex TARGETS="${TARGETS:-x86-64}" -BRANCHES="${BRANCHES:-master}" +export BRANCH="${BRANCH:-master}" export DOCKER_IMAGE="${DOCKER_IMAGE:-openwrt-rootfs}" export DOWNLOAD_FILE="openwrt-*-rootfs.tar.gz" export DOCKERFILE="Dockerfile.rootfs" for TARGET in $TARGETS ; do export TARGET - for BRANCH in $BRANCHES; do - export BRANCH - if [ "$BRANCH" == "master" ]; then - export DOWNLOAD_PATH="snapshots/targets/$(echo $TARGET | tr '-' '/')" - else - export DOWNLOAD_PATH="releases/$BRANCH/targets/$(echo $TARGET | tr '-' '/')" - fi + if [ "$BRANCH" == "master" ]; then + export DOWNLOAD_PATH="snapshots/targets/$(echo $TARGET | tr '-' '/')" + else + export DOWNLOAD_PATH="releases/$BRANCH/targets/$(echo $TARGET | tr '-' '/')" + fi - ./docker-download.sh || exit 1 + ./docker-download.sh || continue - cp -r ./rootfs/* ./build + cp -r ./rootfs/* ./build - ./docker-build.sh || exit 1 - done + ./docker-build.sh || exit 1 done diff --git a/docker-sdk.sh b/docker-sdk.sh index ea67cd0..9e614c5 100755 --- a/docker-sdk.sh +++ b/docker-sdk.sh @@ -3,21 +3,18 @@ set -ex TARGETS="${TARGETS:-x86-64}" -BRANCHES="${BRANCHES:-master}" +export BRANCH="${BRANCH:-master}" export DOCKER_IMAGE="${DOCKER_IMAGE:-openwrt-sdk}" export DOWNLOAD_FILE="openwrt-sdk-*.Linux-x86_64.tar.xz" for TARGET in $TARGETS ; do export TARGET - for BRANCH in $BRANCHES; do - export BRANCH - if [ "$BRANCH" == "master" ]; then - export DOWNLOAD_PATH="snapshots/targets/$(echo $TARGET | tr '-' '/')" - else - export DOWNLOAD_PATH="releases/$BRANCH/targets/$(echo $TARGET | tr '-' '/')" - fi + if [ "$BRANCH" == "master" ]; then + export DOWNLOAD_PATH="snapshots/targets/$(echo $TARGET | tr '-' '/')" + else + export DOWNLOAD_PATH="releases/$BRANCH/targets/$(echo $TARGET | tr '-' '/')" + fi - ./docker-download.sh || exit 1 - ./docker-build.sh || exit 1 - done + ./docker-download.sh || continue + ./docker-build.sh || exit 1 done