From mboxrd@z Thu Jan 1 00:00:00 1970 X-GM-THRID: 6946545331891863552 X-Received: by 2002:a2e:885a:: with SMTP id z26mr8279784ljj.316.1617368620018; Fri, 02 Apr 2021 06:03:40 -0700 (PDT) X-BeenThere: isar-users@googlegroups.com Received: by 2002:a05:6512:3d16:: with SMTP id d22ls854851lfv.1.gmail; Fri, 02 Apr 2021 06:03:39 -0700 (PDT) X-Google-Smtp-Source: ABdhPJw6tii+3ztZ+7lM9ClUqLgTXnxyCwBFsbfL30jsopaJ5/V2qTvP7l5KDjRJQOoTzbB4pDGw X-Received: by 2002:a05:6512:243:: with SMTP id b3mr9091334lfo.529.1617368618994; Fri, 02 Apr 2021 06:03:38 -0700 (PDT) ARC-Seal: i=1; a=rsa-sha256; t=1617368618; cv=none; d=google.com; s=arc-20160816; b=B4jtnGiohY7fsKOW/L12GdpgmjHDTSg+wISQe0Dd7CcPlwTUtSwOoJD/5UC0C8DFAQ UUnDgP6KH/WX5WsdnrYXPXoE0E3X9OLZVE8AIUBLBy/rFBhcXvC7BxlBnGeBEcn/9Ibx 8jaCuSmwO6/vASP+7YjmS2TEqDMrUfYPl/gznOfRK69U6o/rIsYbDutWx1s1OTIrxSYK rKIJ8iR7wEzgPobAUQcRJay0uL7dTJRDpNHjppqftEftaurMG9S0D81EMI/kFTn6jJZU gep0mvSa7p6ENEYVAj5wVg2tHAxVWzNSErtORcqFLHiV+kNEbkSuv/79DkhUFVFKmDqd B1mA== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=content-transfer-encoding:mime-version:message-id:date:subject:cc :to:from; bh=Kl45QS8mmDJu5pGfmahWFPHV+eh1eW9uqhHdCQDy0Cg=; b=qkMv4OJ8x5GKEYRkiyexDulQ0KxiLZAyXAYgNZDkQdE/huVPhX8VCTEPIBPtmwOlWk tdFnmIV4jNceSyQNlV+CjdFk7WdylAphw/UN/DOsevMHLKe2t/M4pY3hgXOqQoICdjuH X6rdAF7vo42ncbJNUiJlHOVFHHedbqVtDw6mUCfz0tN67nYfbGtIYU7N+TBzO48IPJ+r VawVn4Ejw482hyfehf6WmL/zNg53UYcXn+i2lNnUyHJsfa9ZNHxjcBBS05vQz2A61NSe Y+uga40dHf4XO87DaIncKTo1lsy9wgrvfHgMcPZQTiXeOXJrmmCDze9z6ImAAAFeyX8d 9oWA== ARC-Authentication-Results: i=1; gmr-mx.google.com; spf=pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) smtp.mailfrom=amikan@ilbers.de Return-Path: Received: from shymkent.ilbers.de (shymkent.ilbers.de. [85.214.156.166]) by gmr-mx.google.com with ESMTPS id z5si793525ljj.5.2021.04.02.06.03.38 for (version=TLS1_2 cipher=ECDHE-ECDSA-CHACHA20-POLY1305 bits=256/256); Fri, 02 Apr 2021 06:03:38 -0700 (PDT) Received-SPF: pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) client-ip=85.214.156.166; Authentication-Results: gmr-mx.google.com; spf=pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) smtp.mailfrom=amikan@ilbers.de Received: from localhost.localdomain (mm-76-88-121-178.mgts.dynamic.pppoe.byfly.by [178.121.88.76] (may be forged)) (authenticated bits=0) by shymkent.ilbers.de (8.15.2/8.15.2/Debian-8) with ESMTPSA id 132D3bfP006140 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-GCM-SHA384 bits=256 verify=NOT); Fri, 2 Apr 2021 15:03:37 +0200 From: Anton Mikanovich To: isar-users@googlegroups.com Cc: Anton Mikanovich Subject: [PATCH] dpkg: Make mount buildroot reliable Date: Fri, 2 Apr 2021 16:03:30 +0300 Message-Id: <20210402130330.66376-1-amikan@ilbers.de> X-Mailer: git-send-email 2.25.1 MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Spam-Status: No, score=-1.0 required=5.0 tests=ALL_TRUSTED autolearn=unavailable autolearn_force=no version=3.4.2 X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on shymkent.ilbers.de X-TUID: Pq+CD2qVj9N1 We use mounting for several tasks. Mounting and unmounting is performed in shell scripts. If a command fails before unmounting, the directories remain mounted. Implement exception handling around the scripts to ensure that unmounting is performed also in case of script failure. The number of unmounting attempts has been limited to avoid infinite loops. Signed-off-by: Anton Mikanovich --- meta/classes/dpkg-base.bbclass | 93 +++++++++++++++++++--------------- meta/classes/dpkg-gbp.bbclass | 8 +-- meta/classes/dpkg.bbclass | 14 +++-- 3 files changed, 64 insertions(+), 51 deletions(-) diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass index 5c7bddc..544ee36 100644 --- a/meta/classes/dpkg-base.bbclass +++ b/meta/classes/dpkg-base.bbclass @@ -54,13 +54,7 @@ addtask patch before do_adjust_git SRC_APT ?= "" -do_apt_fetch() { - if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then - return 0 - fi - rm -rf ${S} - dpkg_do_mounts - E="${@ isar_export_proxies(d)}" +fetch_apt() { sudo -E chroot ${BUILDCHROOT_DIR} /usr/bin/apt-get update \ -o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \ -o Dir::Etc::SourceParts="-" \ @@ -72,8 +66,27 @@ do_apt_fetch() { sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \ sh -c 'cp /downloads/deb-src/"$1"/"$2"/* ${PP} && cd ${PP} && apt-get -y --only-source source "$2"' my_script "${DISTRO}" "${uri}" done +} + +python do_apt_fetch() { + import glob + import shutil + + src_apt = d.getVar("SRC_APT", True) + if not src_apt: + return 0 - dpkg_undo_mounts + srcsubdir = d.getVar('S', True) + if os.path.exists(srcsubdir): + shutil.rmtree(srcsubdir) + + dpkg_do_mounts(d) + try: + isar_export_proxies(d) + buildchroot = d.getVar('BUILDCHROOT_DIR', True) + bb.build.exec_func("fetch_apt", d) + finally: + dpkg_undo_mounts(d) } addtask apt_fetch after do_unpack before do_patch @@ -118,25 +131,23 @@ do_prepare_build[deptask] = "do_deploy_deb" BUILDROOT = "${BUILDCHROOT_DIR}/${PP}" -dpkg_do_mounts() { - mkdir -p ${BUILDROOT} - sudo mount --bind ${WORKDIR} ${BUILDROOT} - - buildchroot_do_mounts -} - -dpkg_undo_mounts() { - i=1 - while ! sudo umount ${BUILDROOT}; do - sleep 0.1 - i=`expr $i + 1` - if [ $i -gt 100 ]; then - bbwarn "${BUILDROOT}: Couldn't unmount, retrying..." - i=1 - fi - done - sudo rmdir ${BUILDROOT} -} +def dpkg_do_mounts(d): + buildroot = d.getVar('BUILDROOT', True) + workdir = d.getVar("WORKDIR", True) + os.makedirs(buildroot, exist_ok=True) + os.system('sudo mount --bind %s %s' % (workdir, buildroot)) + bb.build.exec_func("buildchroot_do_mounts", d) + +def dpkg_undo_mounts(d): + buildroot = d.getVar('BUILDROOT', True) + for i in range(200): + if not os.system('sudo umount %s' % buildroot): + os.rmdir(buildroot) + return + if i % 100 == 0: + bb.warn("%s: Couldn't unmount, retrying..." % buildroot) + time.sleep(0.1) + bb.fatal("Couldn't unmount, exiting...") # Placeholder for actual dpkg_runbuild() implementation dpkg_runbuild() { @@ -146,10 +157,12 @@ dpkg_runbuild() { python do_dpkg_build() { lock = bb.utils.lockfile(d.getVar("REPO_ISAR_DIR") + "/isar.lock", shared=True) - bb.build.exec_func("dpkg_do_mounts", d) - bb.build.exec_func("dpkg_runbuild", d) - bb.build.exec_func("dpkg_undo_mounts", d) - bb.utils.unlockfile(lock) + dpkg_do_mounts(d) + try: + bb.build.exec_func("dpkg_runbuild", d) + finally: + dpkg_undo_mounts(d) + bb.utils.unlockfile(lock) } addtask dpkg_build before do_build @@ -193,16 +206,16 @@ python do_devshell() { oe_lib_path = os.path.join(d.getVar('LAYERDIR_core'), 'lib') sys.path.insert(0, oe_lib_path) - bb.build.exec_func('dpkg_do_mounts', d) - - isar_export_proxies(d) - - buildchroot = d.getVar('BUILDCHROOT_DIR') - pp_pps = os.path.join(d.getVar('PP'), d.getVar('PPS')) - termcmd = "sudo -E chroot {0} sh -c 'cd {1}; $SHELL -i'" - oe_terminal(termcmd.format(buildchroot, pp_pps), "Isar devshell", d) + dpkg_do_mounts(d) + try: + isar_export_proxies(d) - bb.build.exec_func('dpkg_undo_mounts', d) + buildchroot = d.getVar('BUILDCHROOT_DIR') + pp_pps = os.path.join(d.getVar('PP'), d.getVar('PPS')) + termcmd = "sudo -E chroot {0} sh -c 'cd {1}; $SHELL -i'" + oe_terminal(termcmd.format(buildchroot, pp_pps), "Isar devshell", d) + finally: + dpkg_undo_mounts(d) } addtask devshell after do_prepare_build diff --git a/meta/classes/dpkg-gbp.bbclass b/meta/classes/dpkg-gbp.bbclass index d956e8c..20d2d4c 100644 --- a/meta/classes/dpkg-gbp.bbclass +++ b/meta/classes/dpkg-gbp.bbclass @@ -12,12 +12,7 @@ PATCHTOOL ?= "git" GBP_DEPENDS ?= "git-buildpackage pristine-tar" GBP_EXTRA_OPTIONS ?= "--git-pristine-tar" -do_install_builddeps_append() { - dpkg_do_mounts - distro="${DISTRO}" - if [ ${ISAR_CROSS_COMPILE} -eq 1 ]; then - distro="${HOST_DISTRO}" - fi +builddeps_install_append() { deb_dl_dir_import "${BUILDCHROOT_DIR}" "${distro}" sudo -E chroot ${BUILDCHROOT_DIR} \ apt-get install -y -o Debug::pkgProblemResolver=yes \ @@ -26,7 +21,6 @@ do_install_builddeps_append() { sudo -E chroot ${BUILDCHROOT_DIR} \ apt-get install -y -o Debug::pkgProblemResolver=yes \ --no-install-recommends ${GBP_DEPENDS} - dpkg_undo_mounts } dpkg_runbuild_prepend() { diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass index 4e7c2f7..29e2b89 100644 --- a/meta/classes/dpkg.bbclass +++ b/meta/classes/dpkg.bbclass @@ -6,9 +6,7 @@ inherit dpkg-base PACKAGE_ARCH ?= "${DISTRO_ARCH}" # Install build dependencies for package -do_install_builddeps() { - dpkg_do_mounts - E="${@ isar_export_proxies(d)}" +builddeps_install() { distro="${DISTRO}" if [ ${ISAR_CROSS_COMPILE} -eq 1 ]; then distro="${HOST_DISTRO}" @@ -19,7 +17,15 @@ do_install_builddeps() { deb_dl_dir_export "${BUILDCHROOT_DIR}" "${distro}" sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \ ${PP}/${PPS} ${PACKAGE_ARCH} - dpkg_undo_mounts +} + +python do_install_builddeps() { + dpkg_do_mounts(d) + isar_export_proxies(d) + try: + bb.build.exec_func("builddeps_install", d) + finally: + dpkg_undo_mounts(d) } addtask install_builddeps after do_prepare_build before do_dpkg_build -- 2.20.1