dynamic downstream job creation based on targets

Instead of having a hard coded list of available targets, this approach
makes use os the dumpinfo.pl script that automatically finds available
targets existing in the OpenWrt buildroot. The advantage is to
automatically adapt to available targets in different branches.

Additionally the tagging of SDK containers is improved. While it is
still possible to use SDK based on target tag, it is now also possible
to chose the target based on architecutre. This way less SDK container
are build as one architecutre covers multiple targets.

Signed-off-by: Paul Spooren <mail@aparcar.org>
pull/62/head
Paul Spooren 2020-07-13 18:57:20 -10:00 committed by Petr Štetiar
parent 570ecf2da5
commit ab85512508
9 changed files with 110 additions and 707 deletions

View File

@ -3,10 +3,10 @@ variables:
include:
- local: .gitlab/ci/base.yml
- local: .gitlab/ci/deploy.yml
- local: .gitlab/ci/imagebuilder.yml
- local: .gitlab/ci/sdk.yml
- local: .gitlab/ci/rootfs.yml
- local: .gitlab/ci/targets.yml
- local: .gitlab/ci/targets_rootfs.yml
- remote: https://gitlab.com/ynezz/openwrt-ci/raw/master/openwrt-ci/gitlab/main.yml
@ -15,6 +15,7 @@ stages:
- build-deploy-container
- build
- test
- generate_targets
- deploy
build deploy container:

View File

@ -16,12 +16,21 @@
before_script:
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
.deploy:
image: "$CI_REGISTRY_IMAGE:deploy"
generate-targets:
stage: generate_targets
script: |
git clone --depth 1 https://github.com/openwrt/openwrt.git
cd openwrt/
git checkout "$BRANCH"
bash ../generate_targets.sh
artifacts:
paths:
- targets.yml
trigger-deploy:
stage: deploy
only:
- master
services:
- docker:dind
before_script:
- docker login -u "$DOCKER_USER" -p "$DOCKER_PASS"
trigger:
include:
- artifact: targets.yml
job: generate-targets

View File

@ -0,0 +1,36 @@
.deploy:
image: "$CI_REGISTRY_IMAGE:deploy"
stage: deploy
only:
- master
services:
- docker:dind
before_script:
- docker login -u "$DOCKER_USER" -p "$DOCKER_PASS"
.deploy-rootfs:
extends: .deploy
except:
variables:
- $SKIP_ROOTFS
variables:
DOCKER_IMAGE: "openwrtorg/rootfs"
script:
- bash docker-rootfs.sh
- docker push "$DOCKER_IMAGE"
.deploy-imagebuilder:
extends: .deploy
variables:
DOCKER_IMAGE: "openwrtorg/imagebuilder"
script:
- bash docker-imagebuilder.sh
- docker push "$DOCKER_IMAGE"
.deploy-sdk:
extends: .deploy
variables:
DOCKER_IMAGE: "openwrtorg/sdk"
script:
- bash docker-sdk.sh
- docker push "$DOCKER_IMAGE"

View File

@ -1,10 +1,3 @@
.deploy-imagebuilder:
extends: .deploy
variables:
DOCKER_IMAGE: "openwrtorg/imagebuilder"
script:
- bash docker-imagebuilder.sh
- docker push "$DOCKER_IMAGE"
build-imagebuilder_x86-64:
extends: .build

View File

@ -1,13 +1,3 @@
.deploy-rootfs:
extends: .deploy
except:
variables:
- $SKIP_ROOTFS
variables:
DOCKER_IMAGE: "openwrtorg/rootfs"
script:
- bash docker-rootfs.sh
- docker push "$DOCKER_IMAGE"
build-rootfs_x86-64:
extends: .build

View File

@ -1,10 +1,3 @@
.deploy-sdk:
extends: .deploy
variables:
DOCKER_IMAGE: "openwrtorg/sdk"
script:
- bash docker-sdk.sh
- docker push "$DOCKER_IMAGE"
build-sdk_x86-64:
stage: build

View File

@ -1,672 +0,0 @@
deploy-imagebuilder_apm821xx-nand:
extends: .deploy-imagebuilder
deploy-sdk_apm821xx-nand:
extends: .deploy-sdk
deploy-imagebuilder_apm821xx-sata:
extends: .deploy-imagebuilder
deploy-sdk_apm821xx-sata:
extends: .deploy-sdk
deploy-imagebuilder_ar71xx-generic:
extends: .deploy-imagebuilder
except:
- master
deploy-sdk_ar71xx-generic:
extends: .deploy-sdk
except:
- master
deploy-imagebuilder_ar71xx-mikrotik:
extends: .deploy-imagebuilder
except:
- master
deploy-sdk_ar71xx-mikrotik:
extends: .deploy-sdk
except:
- master
deploy-imagebuilder_ar71xx-nand:
extends: .deploy-imagebuilder
except:
- master
deploy-sdk_ar71xx-nand:
extends: .deploy-sdk
except:
- master
deploy-imagebuilder_ar71xx-tiny:
extends: .deploy-imagebuilder
except:
- master
deploy-sdk_ar71xx-tiny:
extends: .deploy-sdk
except:
- master
deploy-imagebuilder_arc770-generic:
extends: .deploy-imagebuilder
deploy-sdk_arc770-generic:
extends: .deploy-sdk
deploy-imagebuilder_archs38-generic:
extends: .deploy-imagebuilder
deploy-sdk_archs38-generic:
extends: .deploy-sdk
deploy-imagebuilder_armvirt-32:
extends: .deploy-imagebuilder
deploy-sdk_armvirt-32:
extends: .deploy-sdk
deploy-imagebuilder_armvirt-64:
extends: .deploy-imagebuilder
deploy-sdk_armvirt-64:
extends: .deploy-sdk
deploy-imagebuilder_at91-legacy:
extends: .deploy-imagebuilder
deploy-sdk_at91-legacy:
extends: .deploy-sdk
deploy-imagebuilder_at91-sam9x:
extends: .deploy-imagebuilder
deploy-sdk_at91-sam9x:
extends: .deploy-sdk
deploy-imagebuilder_at91-sama5:
extends: .deploy-imagebuilder
deploy-sdk_at91-sama5:
extends: .deploy-sdk
deploy-imagebuilder_at91-sama5d2:
extends: .deploy-imagebuilder
deploy-sdk_at91-sama5d2:
extends: .deploy-sdk
deploy-imagebuilder_at91-sama5d3:
extends: .deploy-imagebuilder
deploy-sdk_at91-sama5d3:
extends: .deploy-sdk
deploy-imagebuilder_at91-sama5d4:
extends: .deploy-imagebuilder
deploy-sdk_at91-sama5d4:
extends: .deploy-sdk
deploy-imagebuilder_ath25-generic:
extends: .deploy-imagebuilder
deploy-sdk_ath25-generic:
extends: .deploy-sdk
deploy-imagebuilder_ath79-generic:
extends: .deploy-imagebuilder
deploy-sdk_ath79-generic:
extends: .deploy-sdk
deploy-imagebuilder_ath79-nand:
extends: .deploy-imagebuilder
deploy-sdk_ath79-nand:
extends: .deploy-sdk
deploy-imagebuilder_ath79-tiny:
extends: .deploy-imagebuilder
deploy-sdk_ath79-tiny:
extends: .deploy-sdk
deploy-imagebuilder_bcm53xx-generic:
extends: .deploy-imagebuilder
deploy-sdk_bcm53xx-generic:
extends: .deploy-sdk
deploy-imagebuilder_brcm2708-bcm2708:
extends: .deploy-imagebuilder
except:
- master
deploy-sdk_brcm2708-bcm2708:
extends: .deploy-sdk
except:
- master
deploy-imagebuilder_bcm27xx-bcm2708:
extends: .deploy-imagebuilder
only:
- master
deploy-sdk_bcm27xx-bcm2708:
extends: .deploy-sdk
only:
- master
deploy-imagebuilder_brcm2708-bcm2709:
extends: .deploy-imagebuilder
except:
- master
deploy-sdk_brcm2708-bcm2709:
extends: .deploy-sdk
except:
- master
deploy-imagebuilder_bcm27xx-bcm2709:
extends: .deploy-imagebuilder
only:
- master
deploy-sdk_bcm27xx-bcm2709:
extends: .deploy-sdk
only:
- master
deploy-imagebuilder_brcm2708-bcm2710:
extends: .deploy-imagebuilder
except:
- master
deploy-sdk_brcm2708-bcm2710:
extends: .deploy-sdk
except:
- master
deploy-imagebuilder_bcm27xx-bcm2710:
extends: .deploy-imagebuilder
only:
- master
deploy-sdk_bcm27xx-bcm2710:
extends: .deploy-sdk
only:
- master
deploy-imagebuilder_bcm27xx-bcm2711:
extends: .deploy-imagebuilder
only:
- master
deploy-sdk_bcm27xx-bcm2711:
extends: .deploy-sdk
only:
- master
deploy-imagebuilder_brcm47xx-generic:
extends: .deploy-imagebuilder
except:
- master
deploy-sdk_brcm47xx-generic:
extends: .deploy-sdk
except:
- master
deploy-imagebuilder_bcm47xx-generic:
extends: .deploy-imagebuilder
only:
- master
deploy-sdk_bcm47xx-generic:
extends: .deploy-sdk
only:
- master
deploy-imagebuilder_brcm47xx-legacy:
extends: .deploy-imagebuilder
except:
- master
deploy-sdk_brcm47xx-legacy:
extends: .deploy-sdk
except:
- master
deploy-imagebuilder_bcm47xx-legacy:
extends: .deploy-imagebuilder
only:
- master
deploy-sdk_bcm47xx-legacy:
extends: .deploy-sdk
only:
- master
deploy-imagebuilder_brcm47xx-mips74k:
extends: .deploy-imagebuilder
except:
- master
deploy-sdk_brcm47xx-mips74k:
extends: .deploy-sdk
except:
- master
deploy-imagebuilder_bcm47xx-mips74k:
extends: .deploy-imagebuilder
only:
- master
deploy-sdk_bcm47xx-mips74k:
extends: .deploy-sdk
only:
- master
deploy-imagebuilder_brcm63xx-generic:
extends: .deploy-imagebuilder
except:
- master
deploy-sdk_brcm63xx-generic:
extends: .deploy-sdk
except:
- master
deploy-imagebuilder_bcm63xx-generic:
extends: .deploy-imagebuilder
only:
- master
deploy-sdk_bcm63xx-generic:
extends: .deploy-sdk
only:
- master
deploy-imagebuilder_brcm63xx-smp:
extends: .deploy-imagebuilder
except:
- master
deploy-sdk_brcm63xx-smp:
extends: .deploy-sdk
except:
- master
deploy-imagebuilder_bcm63xx-smp:
extends: .deploy-imagebuilder
only:
- master
deploy-sdk_bcm63xx-smp:
extends: .deploy-sdk
only:
- master
deploy-imagebuilder_cns3xxx-generic:
extends: .deploy-imagebuilder
deploy-sdk_cns3xxx-generic:
extends: .deploy-sdk
deploy-imagebuilder_gemini-generic:
extends: .deploy-imagebuilder
deploy-sdk_gemini-generic:
extends: .deploy-sdk
deploy-imagebuilder_imx6-generic:
extends: .deploy-imagebuilder
deploy-sdk_imx6-generic:
extends: .deploy-sdk
deploy-imagebuilder_ipq40xx-generic:
extends: .deploy-imagebuilder
deploy-sdk_ipq40xx-generic:
extends: .deploy-sdk
deploy-imagebuilder_ipq806x-generic:
extends: .deploy-imagebuilder
deploy-sdk_ipq806x-generic:
extends: .deploy-sdk
deploy-imagebuilder_kirkwood-generic:
extends: .deploy-imagebuilder
deploy-sdk_kirkwood-generic:
extends: .deploy-sdk
deploy-imagebuilder_lantiq-ase:
extends: .deploy-imagebuilder
deploy-sdk_lantiq-ase:
extends: .deploy-sdk
deploy-imagebuilder_lantiq-falcon:
extends: .deploy-imagebuilder
deploy-sdk_lantiq-falcon:
extends: .deploy-sdk
deploy-imagebuilder_lantiq-xrx200:
extends: .deploy-imagebuilder
deploy-sdk_lantiq-xrx200:
extends: .deploy-sdk
deploy-imagebuilder_lantiq-xway:
extends: .deploy-imagebuilder
deploy-sdk_lantiq-xway:
extends: .deploy-sdk
deploy-imagebuilder_lantiq-xway_legacy:
extends: .deploy-imagebuilder
deploy-sdk_lantiq-xway_legacy:
extends: .deploy-sdk
deploy-imagebuilder_layerscape-armv7:
extends: .deploy-imagebuilder
deploy-sdk_layerscape-armv7:
extends: .deploy-sdk
deploy-imagebuilder_layerscape-armv8_32b:
extends: .deploy-imagebuilder
deploy-sdk_layerscape-armv8_32b:
extends: .deploy-sdk
deploy-imagebuilder_layerscape-armv8_64b:
extends: .deploy-imagebuilder
deploy-sdk_layerscape-armv8_64b:
extends: .deploy-sdk
deploy-imagebuilder_malta-be:
extends: .deploy-imagebuilder
deploy-sdk_malta-be:
extends: .deploy-sdk
deploy-imagebuilder_mediatek-mt7622:
extends: .deploy-imagebuilder
deploy-sdk_mediatek-mt7622:
extends: .deploy-sdk
deploy-imagebuilder_mediatek-mt7623:
extends: .deploy-imagebuilder
deploy-sdk_mediatek-mt7623:
extends: .deploy-sdk
deploy-imagebuilder_mpc85xx-generic:
extends: .deploy-imagebuilder
deploy-sdk_mpc85xx-generic:
extends: .deploy-sdk
deploy-imagebuilder_mpc85xx-p1020:
extends: .deploy-imagebuilder
deploy-sdk_mpc85xx-p1020:
extends: .deploy-sdk
deploy-imagebuilder_mpc85xx-p2020:
extends: .deploy-imagebuilder
deploy-sdk_mpc85xx-p2020:
extends: .deploy-sdk
deploy-imagebuilder_mvebu-cortexa53:
extends: .deploy-imagebuilder
deploy-sdk_mvebu-cortexa53:
extends: .deploy-sdk
deploy-imagebuilder_mvebu-cortexa72:
extends: .deploy-imagebuilder
deploy-sdk_mvebu-cortexa72:
extends: .deploy-sdk
deploy-imagebuilder_mvebu-cortexa9:
extends: .deploy-imagebuilder
deploy-sdk_mvebu-cortexa9:
extends: .deploy-sdk
deploy-imagebuilder_mxs-generic:
extends: .deploy-imagebuilder
deploy-sdk_mxs-generic:
extends: .deploy-sdk
deploy-imagebuilder_octeon-generic:
extends: .deploy-imagebuilder
deploy-sdk_octeon-generic:
extends: .deploy-sdk
deploy-imagebuilder_octeontx-generic:
extends: .deploy-imagebuilder
deploy-sdk_octeontx-generic:
extends: .deploy-sdk
deploy-imagebuilder_omap-generic:
extends: .deploy-imagebuilder
deploy-sdk_omap-generic:
extends: .deploy-sdk
deploy-imagebuilder_oxnas-ox820:
extends: .deploy-imagebuilder
deploy-sdk_oxnas-ox820:
extends: .deploy-sdk
deploy-imagebuilder_pistachio-generic:
extends: .deploy-imagebuilder
deploy-sdk_pistachio-generic:
extends: .deploy-sdk
deploy-imagebuilder_ramips-mt7620:
extends: .deploy-imagebuilder
deploy-sdk_ramips-mt7620:
extends: .deploy-sdk
deploy-imagebuilder_ramips-mt7621:
extends: .deploy-imagebuilder
deploy-sdk_ramips-mt7621:
extends: .deploy-sdk
deploy-imagebuilder_ramips-mt76x8:
extends: .deploy-imagebuilder
deploy-sdk_ramips-mt76x8:
extends: .deploy-sdk
deploy-imagebuilder_ramips-rt288x:
extends: .deploy-imagebuilder
deploy-sdk_ramips-rt288x:
extends: .deploy-sdk
deploy-imagebuilder_ramips-rt305x:
extends: .deploy-imagebuilder
deploy-sdk_ramips-rt305x:
extends: .deploy-sdk
deploy-imagebuilder_ramips-rt3883:
extends: .deploy-imagebuilder
deploy-sdk_ramips-rt3883:
extends: .deploy-sdk
deploy-imagebuilder_rb532-generic:
extends: .deploy-imagebuilder
deploy-sdk_rb532-generic:
extends: .deploy-sdk
deploy-imagebuilder_samsung-s5pv210:
extends: .deploy-imagebuilder
deploy-sdk_samsung-s5pv210:
extends: .deploy-sdk
deploy-imagebuilder_sunxi-cortexa53:
extends: .deploy-imagebuilder
deploy-sdk_sunxi-cortexa53:
extends: .deploy-sdk
deploy-imagebuilder_sunxi-cortexa7:
extends: .deploy-imagebuilder
deploy-sdk_sunxi-cortexa7:
extends: .deploy-sdk
deploy-imagebuilder_sunxi-cortexa8:
extends: .deploy-imagebuilder
deploy-sdk_sunxi-cortexa8:
extends: .deploy-sdk
deploy-imagebuilder_tegra-generic:
extends: .deploy-imagebuilder
deploy-sdk_tegra-generic:
extends: .deploy-sdk
deploy-imagebuilder_x86-64:
extends: .deploy-imagebuilder
deploy-sdk_x86-64:
extends: .deploy-sdk
deploy-imagebuilder_x86-generic:
extends: .deploy-imagebuilder
deploy-sdk_x86-generic:
extends: .deploy-sdk
deploy-imagebuilder_x86-geode:
extends: .deploy-imagebuilder
deploy-sdk_x86-geode:
extends: .deploy-sdk
deploy-imagebuilder_x86-legacy:
extends: .deploy-imagebuilder
deploy-sdk_x86-legacy:
extends: .deploy-sdk
deploy-imagebuilder_zynq-generic:
extends: .deploy-imagebuilder
deploy-sdk_zynq-generic:
extends: .deploy-sdk

View File

@ -2,7 +2,9 @@
set -ex
TARGET=$(echo "$CI_JOB_NAME" | cut -d _ -f 2-)
ARCH=$(echo "$CI_JOB_NAME" | cut -d _ -f 2-)
# shellcheck disable=SC2153
TARGET=$(echo "$TARGETS" | cut -d ' ' -f 1)
export TARGET="${TARGET:-x86-64}"
export BRANCH="${BRANCH:-master}"
export DOCKER_IMAGE="${DOCKER_IMAGE:-openwrt-sdk}"
@ -16,4 +18,26 @@ fi
export DOWNLOAD_PATH
./docker-download.sh || exit 1
./docker-build.sh || exit 1
DOCKERFILE="${DOCKERFILE:-Dockerfile}"
# Copy Dockerfile inside build context to support older Docker versions
# See https://github.com/docker/cli/pull/886
cp "$DOCKERFILE" ./build/
docker build -t "$DOCKER_IMAGE:$ARCH-$BRANCH" -f "./build/$DOCKERFILE" ./build
if [ "$BRANCH" == "master" ]; then
docker tag "$DOCKER_IMAGE:$ARCH-$BRANCH" "$DOCKER_IMAGE:$ARCH"
if [ "$ARCH" == "x86_64" ]; then
docker tag "$DOCKER_IMAGE:$ARCH-$BRANCH" "$DOCKER_IMAGE:latest"
fi
fi
if [ "$ARCH" == "x86_64" ]; then
docker tag "$DOCKER_IMAGE:$ARCH-$BRANCH" "$DOCKER_IMAGE:$BRANCH"
fi
for TARGET_TAG in $TARGETS; do
docker tag "$DOCKER_IMAGE:$ARCH-$BRANCH" "$DOCKER_IMAGE:$TARGET_TAG-$BRANCH"
done
rm -rf ./build

View File

@ -0,0 +1,29 @@
#!/bin/sh
gen_targets_sdk() {
perl ./scripts/dumpinfo.pl architectures | while read -r LINE; do
ARCH=$(echo "$LINE" | cut -d ' ' -f 1)
TARGETS=$(echo "$LINE" | cut -d ' ' -f 2-)
echo "
deploy-sdk_$ARCH:
extends: .deploy-sdk
variables:
TARGETS: $TARGETS"
done
}
gen_targets_imagebuilder() {
perl ./scripts/dumpinfo.pl targets | while read -r LINE; do
TARGET=$(echo "$LINE" | cut -d ' ' -f 1 | tr '/' '-')
echo "
deploy-imagebuilder_$TARGET:
extends: .deploy-imagebuilder"
done
}
echo "include:
- local: .gitlab/ci/deploy.yml
" > ../targets.yml
gen_targets_sdk >> ../targets.yml
gen_targets_imagebuilder >> ../targets.yml
cat ../targets.yml