From mboxrd@z Thu Jan 1 00:00:00 1970 X-GM-THRID: 6953625691033174016 X-Received: by 2002:ac2:54ad:: with SMTP id w13mr15745081lfk.437.1619017147488; Wed, 21 Apr 2021 07:59:07 -0700 (PDT) X-BeenThere: isar-users@googlegroups.com Received: by 2002:ac2:5624:: with SMTP id b4ls1918568lff.0.gmail; Wed, 21 Apr 2021 07:59:06 -0700 (PDT) X-Google-Smtp-Source: ABdhPJyjaP7OftttDj4KuM7ADIvbJsSK8FCQG4XBPQ82HrDZlX/Cg33TfC7gb+D5sCAN8vgTQF0F X-Received: by 2002:a19:f018:: with SMTP id p24mr6255226lfc.421.1619017146515; Wed, 21 Apr 2021 07:59:06 -0700 (PDT) ARC-Seal: i=1; a=rsa-sha256; t=1619017146; cv=none; d=google.com; s=arc-20160816; b=LBu6aSUzaIjbvOte8qzVBm/2Z7vOlTKwLfQNoTmZfmRsajjSwSHdPJxwKHDPykS/Fe qejJqGhjHxVMlV7Dfs6rkr1bF53KQE1hi6in+QVugAFIPUNT6iue4+Sg8OTP8AYttE/n jj1BBQIcFW3zfUTB/7OA5kl4wdih8Y8EDs7D9pyfS9PKkfe59Sz9TP+Lj15dWABwau3W H1n9Rx+Ota0xA5T5lXlTbLEiEkhWBXKyjt+gTFky/5gwvDQ95Z/rjL6T4LwBoKbFtjkx yj1OOV/dVeK5XZ//WRtfDIVZLuBmaCraY6kli69LAu8dCkh88L7EMXpno23cJp2haAc2 0GUA== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from; bh=7A0MA+yvYhViuuA6QzkXoG9E9AVLqVPGVYQlMT+Qz28=; b=0AkZcSwn9UpA4RlKI1O9YCABz9dZkAGtPN6twY6xhIxdLwtVg7AWAw8FV3Cu9XjvPh ZX7gpZOLARvALvgn3vFlvcjX3N8A2Y6T0SNZjcCS0A8OlP60cZDBkk5hULTdTOzNJzee WpU1Qm1IGOKPFDpk53Jr3H1mj2L7CEL027Y2zXPthM7ho2Zzx50QKzMJcYFgI2Ww10+o Vnrw3Bk4mL3FVQhrT7yn2M1m8Gnd+hHDZle7hv0PkVzUrjhU6QC0z/UFC31wOzBlqFmD DQhwTrwcbo833AwG7YeyUR59DJu6LBbjSAaoDDNQH2BgSb0fE+5q9fAXyN6af5phqlk/ BQhw== ARC-Authentication-Results: i=1; gmr-mx.google.com; spf=pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) smtp.mailfrom=amikan@ilbers.de Return-Path: Received: from shymkent.ilbers.de (shymkent.ilbers.de. [85.214.156.166]) by gmr-mx.google.com with ESMTPS id x6si143656ljh.4.2021.04.21.07.59.06 for (version=TLS1_2 cipher=ECDHE-ECDSA-CHACHA20-POLY1305 bits=256/256); Wed, 21 Apr 2021 07:59:06 -0700 (PDT) Received-SPF: pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) client-ip=85.214.156.166; Authentication-Results: gmr-mx.google.com; spf=pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) smtp.mailfrom=amikan@ilbers.de Received: from localhost.localdomain (mm-153-39-214-37.mgts.dynamic.pppoe.byfly.by [37.214.39.153] (may be forged)) (authenticated bits=0) by shymkent.ilbers.de (8.15.2/8.15.2/Debian-8) with ESMTPSA id 13LEx27c027015 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-GCM-SHA384 bits=256 verify=NOT); Wed, 21 Apr 2021 16:59:05 +0200 From: Anton Mikanovich To: isar-users@googlegroups.com Cc: Anton Mikanovich Subject: [PATCH v2 1/5] dpkg: Make mount buildroot reliable Date: Wed, 21 Apr 2021 17:58:51 +0300 Message-Id: <20210421145855.66257-2-amikan@ilbers.de> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20210421145855.66257-1-amikan@ilbers.de> References: <20210421145855.66257-1-amikan@ilbers.de> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Spam-Status: No, score=-1.0 required=5.0 tests=ALL_TRUSTED autolearn=unavailable autolearn_force=no version=3.4.2 X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on shymkent.ilbers.de X-TUID: Vp708qFaTImK We use mounting for several tasks. Mounting and unmounting is performed in shell scripts. If a command fails before unmounting, the directories remain mounted. Implement exception handling around the scripts to ensure that unmounting is performed also in case of script failure. The number of unmounting attempts has been limited to avoid infinite loops. Signed-off-by: Anton Mikanovich --- meta/classes/dpkg-base.bbclass | 126 ++++++++++++++++++++------------- meta/classes/dpkg-gbp.bbclass | 8 +-- meta/classes/dpkg.bbclass | 14 ++-- 3 files changed, 89 insertions(+), 59 deletions(-) diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass index 97661a6..d5e70f1 100644 --- a/meta/classes/dpkg-base.bbclass +++ b/meta/classes/dpkg-base.bbclass @@ -61,12 +61,7 @@ addtask patch before do_adjust_git SRC_APT ?= "" -do_apt_fetch() { - if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then - return 0 - fi - dpkg_do_mounts - E="${@ isar_export_proxies(d)}" +fetch_apt() { sudo -E chroot ${BUILDCHROOT_DIR} /usr/bin/apt-get update \ -o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \ -o Dir::Etc::SourceParts="-" \ @@ -76,21 +71,25 @@ do_apt_fetch() { sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \ sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only-source source "$2"' my_script "${DISTRO}" "${uri}" done +} + +python do_apt_fetch() { + src_apt = d.getVar("SRC_APT", True) + if not src_apt: + return 0 - dpkg_undo_mounts + dpkg_do_mounts(d) + try: + isar_export_proxies(d) + bb.build.exec_func("fetch_apt", d) + finally: + dpkg_undo_mounts(d) } addtask apt_fetch after do_unpack before do_apt_unpack do_apt_fetch[lockfiles] += "${REPO_ISAR_DIR}/isar.lock" -do_apt_unpack() { - if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then - return 0 - fi - rm -rf ${S} - dpkg_do_mounts - E="${@ isar_export_proxies(d)}" - +unpack_apt() { for uri in "${SRC_APT}"; do sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \ sh -c ' \ @@ -101,8 +100,25 @@ do_apt_unpack() { dpkg-source -x "${dscfile}" "${PPS}"' \ my_script "${DISTRO}" "${uri}" done +} - dpkg_undo_mounts +python do_apt_unpack() { + import shutil + + src_apt = d.getVar("SRC_APT", True) + if not src_apt: + return 0 + + srcsubdir = d.getVar('S', True) + if os.path.exists(srcsubdir): + shutil.rmtree(srcsubdir) + + dpkg_do_mounts(d) + try: + isar_export_proxies(d) + bb.build.exec_func("unpack_apt", d) + finally: + dpkg_undo_mounts(d) } addtask apt_unpack after do_apt_fetch before do_patch @@ -146,25 +162,37 @@ do_prepare_build[deptask] = "do_deploy_deb" BUILDROOT = "${BUILDCHROOT_DIR}/${PP}" -dpkg_do_mounts() { - mkdir -p ${BUILDROOT} - sudo mount --bind ${WORKDIR} ${BUILDROOT} - - buildchroot_do_mounts -} - -dpkg_undo_mounts() { - i=1 - while ! sudo umount ${BUILDROOT}; do - sleep 0.1 - i=`expr $i + 1` - if [ $i -gt 100 ]; then - bbwarn "${BUILDROOT}: Couldn't unmount, retrying..." - i=1 - fi - done - sudo rmdir ${BUILDROOT} -} +def ismount(path): + real = os.path.realpath(path) + with open('/proc/mounts') as f: + for line in f.readlines(): + if len(line.split()) > 2 and real == line.split()[1]: + return True + return False + +def dpkg_do_mounts(d): + buildroot = d.getVar('BUILDROOT', True) + if ismount(buildroot): + bb.warn('Path %s already mounted!' % buildroot) + return + workdir = d.getVar('WORKDIR', True) + os.makedirs(buildroot, exist_ok=True) + os.system('sudo mount --bind %s %s' % (workdir, buildroot)) + bb.build.exec_func("buildchroot_do_mounts", d) + +def dpkg_undo_mounts(d): + buildroot = d.getVar('BUILDROOT', True) + if not ismount(buildroot): + bb.warn('Path %s not mounted!' % buildroot) + return + for i in range(200): + if not os.system('sudo umount %s' % buildroot): + os.rmdir(buildroot) + return + if i % 100 == 0: + bb.warn("%s: Couldn't unmount, retrying..." % buildroot) + time.sleep(0.1) + bb.fatal("Couldn't unmount, exiting...") # Placeholder for actual dpkg_runbuild() implementation dpkg_runbuild() { @@ -174,10 +202,12 @@ dpkg_runbuild() { python do_dpkg_build() { lock = bb.utils.lockfile(d.getVar("REPO_ISAR_DIR") + "/isar.lock", shared=True) - bb.build.exec_func("dpkg_do_mounts", d) - bb.build.exec_func("dpkg_runbuild", d) - bb.build.exec_func("dpkg_undo_mounts", d) - bb.utils.unlockfile(lock) + dpkg_do_mounts(d) + try: + bb.build.exec_func("dpkg_runbuild", d) + finally: + dpkg_undo_mounts(d) + bb.utils.unlockfile(lock) } addtask dpkg_build before do_build @@ -221,16 +251,16 @@ python do_devshell() { oe_lib_path = os.path.join(d.getVar('LAYERDIR_core'), 'lib') sys.path.insert(0, oe_lib_path) - bb.build.exec_func('dpkg_do_mounts', d) - - isar_export_proxies(d) - - buildchroot = d.getVar('BUILDCHROOT_DIR') - pp_pps = os.path.join(d.getVar('PP'), d.getVar('PPS')) - termcmd = "sudo -E chroot {0} sh -c 'cd {1}; $SHELL -i'" - oe_terminal(termcmd.format(buildchroot, pp_pps), "Isar devshell", d) + dpkg_do_mounts(d) + try: + isar_export_proxies(d) - bb.build.exec_func('dpkg_undo_mounts', d) + buildchroot = d.getVar('BUILDCHROOT_DIR') + pp_pps = os.path.join(d.getVar('PP'), d.getVar('PPS')) + termcmd = "sudo -E chroot {0} sh -c 'cd {1}; $SHELL -i'" + oe_terminal(termcmd.format(buildchroot, pp_pps), "Isar devshell", d) + finally: + dpkg_undo_mounts(d) } addtask devshell after do_prepare_build diff --git a/meta/classes/dpkg-gbp.bbclass b/meta/classes/dpkg-gbp.bbclass index d956e8c..20d2d4c 100644 --- a/meta/classes/dpkg-gbp.bbclass +++ b/meta/classes/dpkg-gbp.bbclass @@ -12,12 +12,7 @@ PATCHTOOL ?= "git" GBP_DEPENDS ?= "git-buildpackage pristine-tar" GBP_EXTRA_OPTIONS ?= "--git-pristine-tar" -do_install_builddeps_append() { - dpkg_do_mounts - distro="${DISTRO}" - if [ ${ISAR_CROSS_COMPILE} -eq 1 ]; then - distro="${HOST_DISTRO}" - fi +builddeps_install_append() { deb_dl_dir_import "${BUILDCHROOT_DIR}" "${distro}" sudo -E chroot ${BUILDCHROOT_DIR} \ apt-get install -y -o Debug::pkgProblemResolver=yes \ @@ -26,7 +21,6 @@ do_install_builddeps_append() { sudo -E chroot ${BUILDCHROOT_DIR} \ apt-get install -y -o Debug::pkgProblemResolver=yes \ --no-install-recommends ${GBP_DEPENDS} - dpkg_undo_mounts } dpkg_runbuild_prepend() { diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass index 4e7c2f7..29e2b89 100644 --- a/meta/classes/dpkg.bbclass +++ b/meta/classes/dpkg.bbclass @@ -6,9 +6,7 @@ inherit dpkg-base PACKAGE_ARCH ?= "${DISTRO_ARCH}" # Install build dependencies for package -do_install_builddeps() { - dpkg_do_mounts - E="${@ isar_export_proxies(d)}" +builddeps_install() { distro="${DISTRO}" if [ ${ISAR_CROSS_COMPILE} -eq 1 ]; then distro="${HOST_DISTRO}" @@ -19,7 +17,15 @@ do_install_builddeps() { deb_dl_dir_export "${BUILDCHROOT_DIR}" "${distro}" sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \ ${PP}/${PPS} ${PACKAGE_ARCH} - dpkg_undo_mounts +} + +python do_install_builddeps() { + dpkg_do_mounts(d) + isar_export_proxies(d) + try: + bb.build.exec_func("builddeps_install", d) + finally: + dpkg_undo_mounts(d) } addtask install_builddeps after do_prepare_build before do_dpkg_build -- 2.25.1