From mboxrd@z Thu Jan 1 00:00:00 1970 X-GM-THRID: 6959801776553852928 X-Received: by 2002:a05:6402:31b1:: with SMTP id dj17mr16199308edb.7.1620455130234; Fri, 07 May 2021 23:25:30 -0700 (PDT) X-BeenThere: isar-users@googlegroups.com Received: by 2002:a17:907:94d4:: with SMTP id dn20ls21190ejc.6.gmail; Fri, 07 May 2021 23:25:29 -0700 (PDT) X-Google-Smtp-Source: ABdhPJxL2HBjUPx+S8rwJXQMIaRv8z/WiTaDejUqXJAGVTvMBsj9qbzNFkt6AFualPgLETut9nfP X-Received: by 2002:a17:906:49c1:: with SMTP id w1mr14669579ejv.178.1620455129239; Fri, 07 May 2021 23:25:29 -0700 (PDT) ARC-Seal: i=1; a=rsa-sha256; t=1620455129; cv=none; d=google.com; s=arc-20160816; b=xvpq/A+O0pngyxP5M9hX9F+X1yUo7rTWOdySg6KNfHg3iXKdFIfB4wJUcOWgcK0QQ+ 3aqaSQ33s9V1lVSmWNrbKd/s8iaF553Dx4ALNMEjVDzC9yEqlSLhEZWw1Av1fpOl27p6 RWzlNw7y4NJQzeXJzu21ZDEDjWc9SDrKZA56hqFQYwn2G4AnvGJFTSK5m6wInRAQDrH4 uXpkKcFvcmb0jzGY35S7IifUu2NHAAWC2baJFyin3rW/lYkF2HBB0moDJ/s4CX3F6ey4 TbqcSIkcaWjMsRYRLouzC/i+n0kMxIbPn/4PmB+CvJy4AbEZvwM5zYXQO/Msn1f0BLFa J5fQ== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from; bh=7A0MA+yvYhViuuA6QzkXoG9E9AVLqVPGVYQlMT+Qz28=; b=Dt6ArGnpUNvVlBxoe/GSGf+JI9D6vVfK2rEkecT0VoIzErh4lqlfbR9uqt0spSlkam awvsXmJ7H33PDg9zXQxO8mFWfSYlcawpXnUIil0yNMzfv2S4nDs2tCxm9iov3WCVG5Mr Ri2AtBu0sqjp3/4MnK2Agz6WKxLRu/XVgPB9pF12nA4vwaS2iBVGQu/c5V4izehUJ2DY U3y0/ZjlxJ1SkFy++9RNdGDcRDpkRtvgNd2qkz02mfpNVzCWC+QYhZvPZD3C4B4QK7i4 jpv00fRs0B0YwVQx0uO4fz9a5C0Q1/ceo8V3R9kZRd+QWEaoLuHBR3Vbnppl2uyjTKb1 T7Hg== ARC-Authentication-Results: i=1; gmr-mx.google.com; spf=pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) smtp.mailfrom=amikan@ilbers.de Return-Path: Received: from shymkent.ilbers.de (shymkent.ilbers.de. [85.214.156.166]) by gmr-mx.google.com with ESMTPS id n3si257014edv.4.2021.05.07.23.25.29 for (version=TLS1_2 cipher=ECDHE-ECDSA-CHACHA20-POLY1305 bits=256/256); Fri, 07 May 2021 23:25:29 -0700 (PDT) Received-SPF: pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) client-ip=85.214.156.166; Authentication-Results: gmr-mx.google.com; spf=pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) smtp.mailfrom=amikan@ilbers.de Received: from localhost.localdomain (mm-99-50-214-37.mgts.dynamic.pppoe.byfly.by [37.214.50.99] (may be forged)) (authenticated bits=0) by shymkent.ilbers.de (8.15.2/8.15.2/Debian-8) with ESMTPSA id 1486PPYi003793 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-GCM-SHA384 bits=256 verify=NOT); Sat, 8 May 2021 08:25:28 +0200 From: Anton Mikanovich To: isar-users@googlegroups.com Cc: Anton Mikanovich Subject: [PATCH v3 1/5] dpkg: Make mount buildroot reliable Date: Sat, 8 May 2021 09:25:14 +0300 Message-Id: <20210508062518.83852-2-amikan@ilbers.de> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20210508062518.83852-1-amikan@ilbers.de> References: <20210508062518.83852-1-amikan@ilbers.de> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Spam-Status: No, score=-1.0 required=5.0 tests=ALL_TRUSTED autolearn=unavailable autolearn_force=no version=3.4.2 X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on shymkent.ilbers.de X-TUID: 7C613BgCZhAN We use mounting for several tasks. Mounting and unmounting is performed in shell scripts. If a command fails before unmounting, the directories remain mounted. Implement exception handling around the scripts to ensure that unmounting is performed also in case of script failure. The number of unmounting attempts has been limited to avoid infinite loops. Signed-off-by: Anton Mikanovich --- meta/classes/dpkg-base.bbclass | 126 ++++++++++++++++++++------------- meta/classes/dpkg-gbp.bbclass | 8 +-- meta/classes/dpkg.bbclass | 14 ++-- 3 files changed, 89 insertions(+), 59 deletions(-) diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass index 97661a6..d5e70f1 100644 --- a/meta/classes/dpkg-base.bbclass +++ b/meta/classes/dpkg-base.bbclass @@ -61,12 +61,7 @@ addtask patch before do_adjust_git SRC_APT ?= "" -do_apt_fetch() { - if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then - return 0 - fi - dpkg_do_mounts - E="${@ isar_export_proxies(d)}" +fetch_apt() { sudo -E chroot ${BUILDCHROOT_DIR} /usr/bin/apt-get update \ -o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \ -o Dir::Etc::SourceParts="-" \ @@ -76,21 +71,25 @@ do_apt_fetch() { sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \ sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only-source source "$2"' my_script "${DISTRO}" "${uri}" done +} + +python do_apt_fetch() { + src_apt = d.getVar("SRC_APT", True) + if not src_apt: + return 0 - dpkg_undo_mounts + dpkg_do_mounts(d) + try: + isar_export_proxies(d) + bb.build.exec_func("fetch_apt", d) + finally: + dpkg_undo_mounts(d) } addtask apt_fetch after do_unpack before do_apt_unpack do_apt_fetch[lockfiles] += "${REPO_ISAR_DIR}/isar.lock" -do_apt_unpack() { - if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then - return 0 - fi - rm -rf ${S} - dpkg_do_mounts - E="${@ isar_export_proxies(d)}" - +unpack_apt() { for uri in "${SRC_APT}"; do sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \ sh -c ' \ @@ -101,8 +100,25 @@ do_apt_unpack() { dpkg-source -x "${dscfile}" "${PPS}"' \ my_script "${DISTRO}" "${uri}" done +} - dpkg_undo_mounts +python do_apt_unpack() { + import shutil + + src_apt = d.getVar("SRC_APT", True) + if not src_apt: + return 0 + + srcsubdir = d.getVar('S', True) + if os.path.exists(srcsubdir): + shutil.rmtree(srcsubdir) + + dpkg_do_mounts(d) + try: + isar_export_proxies(d) + bb.build.exec_func("unpack_apt", d) + finally: + dpkg_undo_mounts(d) } addtask apt_unpack after do_apt_fetch before do_patch @@ -146,25 +162,37 @@ do_prepare_build[deptask] = "do_deploy_deb" BUILDROOT = "${BUILDCHROOT_DIR}/${PP}" -dpkg_do_mounts() { - mkdir -p ${BUILDROOT} - sudo mount --bind ${WORKDIR} ${BUILDROOT} - - buildchroot_do_mounts -} - -dpkg_undo_mounts() { - i=1 - while ! sudo umount ${BUILDROOT}; do - sleep 0.1 - i=`expr $i + 1` - if [ $i -gt 100 ]; then - bbwarn "${BUILDROOT}: Couldn't unmount, retrying..." - i=1 - fi - done - sudo rmdir ${BUILDROOT} -} +def ismount(path): + real = os.path.realpath(path) + with open('/proc/mounts') as f: + for line in f.readlines(): + if len(line.split()) > 2 and real == line.split()[1]: + return True + return False + +def dpkg_do_mounts(d): + buildroot = d.getVar('BUILDROOT', True) + if ismount(buildroot): + bb.warn('Path %s already mounted!' % buildroot) + return + workdir = d.getVar('WORKDIR', True) + os.makedirs(buildroot, exist_ok=True) + os.system('sudo mount --bind %s %s' % (workdir, buildroot)) + bb.build.exec_func("buildchroot_do_mounts", d) + +def dpkg_undo_mounts(d): + buildroot = d.getVar('BUILDROOT', True) + if not ismount(buildroot): + bb.warn('Path %s not mounted!' % buildroot) + return + for i in range(200): + if not os.system('sudo umount %s' % buildroot): + os.rmdir(buildroot) + return + if i % 100 == 0: + bb.warn("%s: Couldn't unmount, retrying..." % buildroot) + time.sleep(0.1) + bb.fatal("Couldn't unmount, exiting...") # Placeholder for actual dpkg_runbuild() implementation dpkg_runbuild() { @@ -174,10 +202,12 @@ dpkg_runbuild() { python do_dpkg_build() { lock = bb.utils.lockfile(d.getVar("REPO_ISAR_DIR") + "/isar.lock", shared=True) - bb.build.exec_func("dpkg_do_mounts", d) - bb.build.exec_func("dpkg_runbuild", d) - bb.build.exec_func("dpkg_undo_mounts", d) - bb.utils.unlockfile(lock) + dpkg_do_mounts(d) + try: + bb.build.exec_func("dpkg_runbuild", d) + finally: + dpkg_undo_mounts(d) + bb.utils.unlockfile(lock) } addtask dpkg_build before do_build @@ -221,16 +251,16 @@ python do_devshell() { oe_lib_path = os.path.join(d.getVar('LAYERDIR_core'), 'lib') sys.path.insert(0, oe_lib_path) - bb.build.exec_func('dpkg_do_mounts', d) - - isar_export_proxies(d) - - buildchroot = d.getVar('BUILDCHROOT_DIR') - pp_pps = os.path.join(d.getVar('PP'), d.getVar('PPS')) - termcmd = "sudo -E chroot {0} sh -c 'cd {1}; $SHELL -i'" - oe_terminal(termcmd.format(buildchroot, pp_pps), "Isar devshell", d) + dpkg_do_mounts(d) + try: + isar_export_proxies(d) - bb.build.exec_func('dpkg_undo_mounts', d) + buildchroot = d.getVar('BUILDCHROOT_DIR') + pp_pps = os.path.join(d.getVar('PP'), d.getVar('PPS')) + termcmd = "sudo -E chroot {0} sh -c 'cd {1}; $SHELL -i'" + oe_terminal(termcmd.format(buildchroot, pp_pps), "Isar devshell", d) + finally: + dpkg_undo_mounts(d) } addtask devshell after do_prepare_build diff --git a/meta/classes/dpkg-gbp.bbclass b/meta/classes/dpkg-gbp.bbclass index d956e8c..20d2d4c 100644 --- a/meta/classes/dpkg-gbp.bbclass +++ b/meta/classes/dpkg-gbp.bbclass @@ -12,12 +12,7 @@ PATCHTOOL ?= "git" GBP_DEPENDS ?= "git-buildpackage pristine-tar" GBP_EXTRA_OPTIONS ?= "--git-pristine-tar" -do_install_builddeps_append() { - dpkg_do_mounts - distro="${DISTRO}" - if [ ${ISAR_CROSS_COMPILE} -eq 1 ]; then - distro="${HOST_DISTRO}" - fi +builddeps_install_append() { deb_dl_dir_import "${BUILDCHROOT_DIR}" "${distro}" sudo -E chroot ${BUILDCHROOT_DIR} \ apt-get install -y -o Debug::pkgProblemResolver=yes \ @@ -26,7 +21,6 @@ do_install_builddeps_append() { sudo -E chroot ${BUILDCHROOT_DIR} \ apt-get install -y -o Debug::pkgProblemResolver=yes \ --no-install-recommends ${GBP_DEPENDS} - dpkg_undo_mounts } dpkg_runbuild_prepend() { diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass index 4e7c2f7..29e2b89 100644 --- a/meta/classes/dpkg.bbclass +++ b/meta/classes/dpkg.bbclass @@ -6,9 +6,7 @@ inherit dpkg-base PACKAGE_ARCH ?= "${DISTRO_ARCH}" # Install build dependencies for package -do_install_builddeps() { - dpkg_do_mounts - E="${@ isar_export_proxies(d)}" +builddeps_install() { distro="${DISTRO}" if [ ${ISAR_CROSS_COMPILE} -eq 1 ]; then distro="${HOST_DISTRO}" @@ -19,7 +17,15 @@ do_install_builddeps() { deb_dl_dir_export "${BUILDCHROOT_DIR}" "${distro}" sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \ ${PP}/${PPS} ${PACKAGE_ARCH} - dpkg_undo_mounts +} + +python do_install_builddeps() { + dpkg_do_mounts(d) + isar_export_proxies(d) + try: + bb.build.exec_func("builddeps_install", d) + finally: + dpkg_undo_mounts(d) } addtask install_builddeps after do_prepare_build before do_dpkg_build -- 2.25.1