Re: [PATCH v3 1/5] dpkg: Make mount buildroot reliable

[Henning Schild <henning.schild@siemens.com>]

Am Sat,  8 May 2021 09:25:14 +0300
schrieb Anton Mikanovich <amikan@ilbers.de>:

> We use mounting for several tasks. Mounting and unmounting is
> performed in shell scripts. If a command fails before unmounting, the
> directories remain mounted.
> 
> Implement exception handling around the scripts to ensure that
> unmounting is performed also in case of script failure. The number of
> unmounting attempts has been limited to avoid infinite loops.
> 
> Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
> ---
>  meta/classes/dpkg-base.bbclass | 126
> ++++++++++++++++++++------------- meta/classes/dpkg-gbp.bbclass  |
> 8 +-- meta/classes/dpkg.bbclass      |  14 ++--
>  3 files changed, 89 insertions(+), 59 deletions(-)
> 
> diff --git a/meta/classes/dpkg-base.bbclass
> b/meta/classes/dpkg-base.bbclass index 97661a6..d5e70f1 100644
> --- a/meta/classes/dpkg-base.bbclass
> +++ b/meta/classes/dpkg-base.bbclass
> @@ -61,12 +61,7 @@ addtask patch before do_adjust_git
>  
>  SRC_APT ?= ""
>  
> -do_apt_fetch() {
> -    if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then
> -        return 0
> -    fi
> -    dpkg_do_mounts
> -    E="${@ isar_export_proxies(d)}"

Sure that the new pattern will make this shell have the variables?
Building behind proxies is unfortunately something we need. Could be
tried with BB_NO_NETWORK which should make you fail because of
deadend_proxy

Henning

> +fetch_apt() {
>      sudo -E chroot ${BUILDCHROOT_DIR} /usr/bin/apt-get update \
>          -o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \
>          -o Dir::Etc::SourceParts="-" \
> @@ -76,21 +71,25 @@ do_apt_fetch() {
>          sudo -E chroot --userspec=$( id -u ):$( id -g )
> ${BUILDCHROOT_DIR} \ sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" &&
> cd /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only
> --only-source source "$2"' my_script "${DISTRO}" "${uri}" done +}
> +
> +python do_apt_fetch() {
> +    src_apt = d.getVar("SRC_APT", True)
> +    if not src_apt:
> +        return 0
>  
> -    dpkg_undo_mounts
> +    dpkg_do_mounts(d)
> +    try:
> +        isar_export_proxies(d)
> +        bb.build.exec_func("fetch_apt", d)
> +    finally:
> +        dpkg_undo_mounts(d)
>  }
>  
>  addtask apt_fetch after do_unpack before do_apt_unpack
>  do_apt_fetch[lockfiles] += "${REPO_ISAR_DIR}/isar.lock"
>  
> -do_apt_unpack() {
> -    if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then
> -        return 0
> -    fi
> -    rm -rf ${S}
> -    dpkg_do_mounts
> -    E="${@ isar_export_proxies(d)}"
> -
> +unpack_apt() {
>      for uri in "${SRC_APT}"; do
>          sudo -E chroot --userspec=$( id -u ):$( id -g )
> ${BUILDCHROOT_DIR} \ sh -c ' \
> @@ -101,8 +100,25 @@ do_apt_unpack() {
>                  dpkg-source -x "${dscfile}" "${PPS}"' \
>                      my_script "${DISTRO}" "${uri}"
>      done
> +}
>  
> -    dpkg_undo_mounts
> +python do_apt_unpack() {
> +    import shutil
> +
> +    src_apt = d.getVar("SRC_APT", True)
> +    if not src_apt:
> +        return 0
> +
> +    srcsubdir = d.getVar('S', True)
> +    if os.path.exists(srcsubdir):
> +        shutil.rmtree(srcsubdir)
> +
> +    dpkg_do_mounts(d)
> +    try:
> +        isar_export_proxies(d)
> +        bb.build.exec_func("unpack_apt", d)
> +    finally:
> +        dpkg_undo_mounts(d)
>  }
>  
>  addtask apt_unpack after do_apt_fetch before do_patch
> @@ -146,25 +162,37 @@ do_prepare_build[deptask] = "do_deploy_deb"
>  
>  BUILDROOT = "${BUILDCHROOT_DIR}/${PP}"
>  
> -dpkg_do_mounts() {
> -    mkdir -p ${BUILDROOT}
> -    sudo mount --bind ${WORKDIR} ${BUILDROOT}
> -
> -    buildchroot_do_mounts
> -}
> -
> -dpkg_undo_mounts() {
> -    i=1
> -    while ! sudo umount ${BUILDROOT}; do
> -        sleep 0.1
> -        i=`expr $i + 1`
> -        if [ $i -gt 100 ]; then
> -            bbwarn "${BUILDROOT}: Couldn't unmount, retrying..."
> -            i=1
> -        fi
> -    done
> -    sudo rmdir ${BUILDROOT}
> -}
> +def ismount(path):
> +    real = os.path.realpath(path)
> +    with open('/proc/mounts') as f:
> +        for line in f.readlines():
> +            if len(line.split()) > 2 and real == line.split()[1]:
> +                return True
> +    return False
> +
> +def dpkg_do_mounts(d):
> +    buildroot = d.getVar('BUILDROOT', True)
> +    if ismount(buildroot):
> +        bb.warn('Path %s already mounted!' % buildroot)
> +        return
> +    workdir = d.getVar('WORKDIR', True)
> +    os.makedirs(buildroot, exist_ok=True)
> +    os.system('sudo mount --bind %s %s' % (workdir, buildroot))
> +    bb.build.exec_func("buildchroot_do_mounts", d)
> +
> +def dpkg_undo_mounts(d):
> +    buildroot = d.getVar('BUILDROOT', True)
> +    if not ismount(buildroot):
> +        bb.warn('Path %s not mounted!' % buildroot)
> +        return
> +    for i in range(200):
> +        if not os.system('sudo umount %s' % buildroot):
> +            os.rmdir(buildroot)
> +            return
> +        if i % 100 == 0:
> +            bb.warn("%s: Couldn't unmount, retrying..." % buildroot)
> +        time.sleep(0.1)
> +    bb.fatal("Couldn't unmount, exiting...")
>  
>  # Placeholder for actual dpkg_runbuild() implementation
>  dpkg_runbuild() {
> @@ -174,10 +202,12 @@ dpkg_runbuild() {
>  python do_dpkg_build() {
>      lock = bb.utils.lockfile(d.getVar("REPO_ISAR_DIR") +
> "/isar.lock", shared=True)
> -    bb.build.exec_func("dpkg_do_mounts", d)
> -    bb.build.exec_func("dpkg_runbuild", d)
> -    bb.build.exec_func("dpkg_undo_mounts", d)
> -    bb.utils.unlockfile(lock)
> +    dpkg_do_mounts(d)
> +    try:
> +        bb.build.exec_func("dpkg_runbuild", d)
> +    finally:
> +        dpkg_undo_mounts(d)
> +        bb.utils.unlockfile(lock)
>  }
>  
>  addtask dpkg_build before do_build
> @@ -221,16 +251,16 @@ python do_devshell() {
>      oe_lib_path = os.path.join(d.getVar('LAYERDIR_core'), 'lib')
>      sys.path.insert(0, oe_lib_path)
>  
> -    bb.build.exec_func('dpkg_do_mounts', d)
> -
> -    isar_export_proxies(d)
> -
> -    buildchroot = d.getVar('BUILDCHROOT_DIR')
> -    pp_pps = os.path.join(d.getVar('PP'), d.getVar('PPS'))
> -    termcmd = "sudo -E chroot {0} sh -c 'cd {1}; $SHELL -i'"
> -    oe_terminal(termcmd.format(buildchroot, pp_pps), "Isar
> devshell", d)
> +    dpkg_do_mounts(d)
> +    try:
> +        isar_export_proxies(d)
>  
> -    bb.build.exec_func('dpkg_undo_mounts', d)
> +        buildchroot = d.getVar('BUILDCHROOT_DIR')
> +        pp_pps = os.path.join(d.getVar('PP'), d.getVar('PPS'))
> +        termcmd = "sudo -E chroot {0} sh -c 'cd {1}; $SHELL -i'"
> +        oe_terminal(termcmd.format(buildchroot, pp_pps), "Isar
> devshell", d)
> +    finally:
> +        dpkg_undo_mounts(d)
>  }
>  
>  addtask devshell after do_prepare_build
> diff --git a/meta/classes/dpkg-gbp.bbclass
> b/meta/classes/dpkg-gbp.bbclass index d956e8c..20d2d4c 100644
> --- a/meta/classes/dpkg-gbp.bbclass
> +++ b/meta/classes/dpkg-gbp.bbclass
> @@ -12,12 +12,7 @@ PATCHTOOL ?= "git"
>  GBP_DEPENDS ?= "git-buildpackage pristine-tar"
>  GBP_EXTRA_OPTIONS ?= "--git-pristine-tar"
>  
> -do_install_builddeps_append() {
> -    dpkg_do_mounts
> -    distro="${DISTRO}"
> -    if [ ${ISAR_CROSS_COMPILE} -eq 1 ]; then
> -       distro="${HOST_DISTRO}"
> -    fi
> +builddeps_install_append() {
>      deb_dl_dir_import "${BUILDCHROOT_DIR}" "${distro}"
>      sudo -E chroot ${BUILDCHROOT_DIR} \
>          apt-get install -y -o Debug::pkgProblemResolver=yes \
> @@ -26,7 +21,6 @@ do_install_builddeps_append() {
>      sudo -E chroot ${BUILDCHROOT_DIR} \
>          apt-get install -y -o Debug::pkgProblemResolver=yes \
>                          --no-install-recommends ${GBP_DEPENDS}
> -    dpkg_undo_mounts
>  }
>  
>  dpkg_runbuild_prepend() {
> diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
> index 4e7c2f7..29e2b89 100644
> --- a/meta/classes/dpkg.bbclass
> +++ b/meta/classes/dpkg.bbclass
> @@ -6,9 +6,7 @@ inherit dpkg-base
>  PACKAGE_ARCH ?= "${DISTRO_ARCH}"
>  
>  # Install build dependencies for package
> -do_install_builddeps() {
> -    dpkg_do_mounts
> -    E="${@ isar_export_proxies(d)}"
> +builddeps_install() {
>      distro="${DISTRO}"
>      if [ ${ISAR_CROSS_COMPILE} -eq 1 ]; then
>         distro="${HOST_DISTRO}"
> @@ -19,7 +17,15 @@ do_install_builddeps() {
>      deb_dl_dir_export "${BUILDCHROOT_DIR}" "${distro}"
>      sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
>          ${PP}/${PPS} ${PACKAGE_ARCH}
> -    dpkg_undo_mounts
> +}
> +
> +python do_install_builddeps() {
> +    dpkg_do_mounts(d)
> +    isar_export_proxies(d)
> +    try:
> +        bb.build.exec_func("builddeps_install", d)
> +    finally:
> +        dpkg_undo_mounts(d)
>  }
>  
>  addtask install_builddeps after do_prepare_build before do_dpkg_build