From mboxrd@z Thu Jan 1 00:00:00 1970 X-GM-THRID: 6946545331891863552 X-Received: by 2002:a1c:f701:: with SMTP id v1mr6679540wmh.69.1617863251183; Wed, 07 Apr 2021 23:27:31 -0700 (PDT) X-BeenThere: isar-users@googlegroups.com Received: by 2002:a5d:5348:: with SMTP id t8ls1848122wrv.2.gmail; Wed, 07 Apr 2021 23:27:30 -0700 (PDT) X-Google-Smtp-Source: ABdhPJyxKAmfP1Oj6Hou4BHvfJTjowFsJrrCN0Hbp50+J5dDjzLa+k1xKafEUwceitwYqAD4ICwB X-Received: by 2002:adf:f60e:: with SMTP id t14mr9161689wrp.51.1617863250293; Wed, 07 Apr 2021 23:27:30 -0700 (PDT) ARC-Seal: i=1; a=rsa-sha256; t=1617863250; cv=none; d=google.com; s=arc-20160816; b=hW8CoRv9DHhH+9wYJk2H024VowIvkGn/jsLC30dy4+5FyJLs0Im5yirAAx9hj76+hO ZE77YnkvEzLKeOByvqDaS6HRknAxKK8zhKmYwJsScYDuXhkZ22n3Cy6okCyDab1mdwda j72A2fb6DPHmoM4y59jbGOF3yGzu2Nov8h/4avo9KBV+4qF4c8AEzg8D7JpDryljPZeL ayC5Z7vdrQlzcRuaV22B3x0Hk7R7/txGJizenZm/6i2BjEXIqwppTvjuVvQcUvWHdD9E d8dW4KoAGdXRLHQPD35/NmKCvXOpqZYTqQZpd+iIAzNg7RPg+bZFbbkACKTMl8Cavf8l tRKQ== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=content-transfer-encoding:content-language:in-reply-to:mime-version :user-agent:date:message-id:from:references:to:subject; bh=SuU3LMNwMYtUPbMi+96Y7mkypHcdqKAcaYPQ5Fqj5tQ=; b=A3TQpVdK+QKeHINn+izpPg+LhEX2Mu4cNLFzqCugaS70hmXTOpDRFNHXGcFrxvdnJC E7uFKOEQdcQpBojEPEEuUM5dfi72iqG3WU5wHWJWagBzl/yNWPBsMeNS3K8/itj6v4Wq HihHlM9jWSr3/H8Rs5wGzQvxbjmFLPBs78ymNIWhPqx4EnXZe7oRYwSzyd916yPMU70i xNBUy90EEoe3FVN81FVjkzQeqtvXYkkPTw1XllG8bqrxRdgxSsDsJ7BYG0bC5KP9HDS8 yFZspGh0aL2gjxQQoKrONWSV09iOyX53inxDqnpIUdBk0NaejSuSyQTzPFvNXP1SsVcH po1A== ARC-Authentication-Results: i=1; gmr-mx.google.com; spf=pass (google.com: domain of jan.kiszka@siemens.com designates 194.138.37.39 as permitted sender) smtp.mailfrom=jan.kiszka@siemens.com; dmarc=pass (p=NONE sp=NONE dis=NONE) header.from=siemens.com Return-Path: Received: from lizzard.sbs.de (lizzard.sbs.de. [194.138.37.39]) by gmr-mx.google.com with ESMTPS id k4si1639389wrc.0.2021.04.07.23.27.30 for (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Wed, 07 Apr 2021 23:27:30 -0700 (PDT) Received-SPF: pass (google.com: domain of jan.kiszka@siemens.com designates 194.138.37.39 as permitted sender) client-ip=194.138.37.39; Authentication-Results: gmr-mx.google.com; spf=pass (google.com: domain of jan.kiszka@siemens.com designates 194.138.37.39 as permitted sender) smtp.mailfrom=jan.kiszka@siemens.com; dmarc=pass (p=NONE sp=NONE dis=NONE) header.from=siemens.com Received: from mail2.sbs.de (mail2.sbs.de [192.129.41.66]) by lizzard.sbs.de (8.15.2/8.15.2) with ESMTPS id 1386RThZ021099 (version=TLSv1.2 cipher=DHE-RSA-AES256-GCM-SHA384 bits=256 verify=OK); Thu, 8 Apr 2021 08:27:29 +0200 Received: from [167.87.250.28] ([167.87.250.28]) by mail2.sbs.de (8.15.2/8.15.2) with ESMTP id 1386RTpi005320; Thu, 8 Apr 2021 08:27:29 +0200 Subject: Re: [PATCH] dpkg: Make mount buildroot reliable To: Anton Mikanovich , isar-users@googlegroups.com References: <20210402130330.66376-1-amikan@ilbers.de> From: Jan Kiszka Message-ID: <82deddfc-687d-1627-37f9-c3c0eae82505@siemens.com> Date: Thu, 8 Apr 2021 08:27:29 +0200 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101 Thunderbird/78.8.0 MIME-Version: 1.0 In-Reply-To: <20210402130330.66376-1-amikan@ilbers.de> Content-Type: text/plain; charset=utf-8 Content-Language: en-US Content-Transfer-Encoding: 7bit X-TUID: 3oTXDvgsCh5j On 02.04.21 15:03, Anton Mikanovich wrote: > We use mounting for several tasks. Mounting and unmounting is performed > in shell scripts. If a command fails before unmounting, the directories > remain mounted. Right - but what is the problem to be solved? We are cleaning up in build_completed() whatever is left by individual jobs. I'm not against this refactoring, but the purpose should be made clearer. Jan > > Implement exception handling around the scripts to ensure that > unmounting is performed also in case of script failure. The number of > unmounting attempts has been limited to avoid infinite loops. > > Signed-off-by: Anton Mikanovich > --- > meta/classes/dpkg-base.bbclass | 93 +++++++++++++++++++--------------- > meta/classes/dpkg-gbp.bbclass | 8 +-- > meta/classes/dpkg.bbclass | 14 +++-- > 3 files changed, 64 insertions(+), 51 deletions(-) > > diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass > index 5c7bddc..544ee36 100644 > --- a/meta/classes/dpkg-base.bbclass > +++ b/meta/classes/dpkg-base.bbclass > @@ -54,13 +54,7 @@ addtask patch before do_adjust_git > > SRC_APT ?= "" > > -do_apt_fetch() { > - if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then > - return 0 > - fi > - rm -rf ${S} > - dpkg_do_mounts > - E="${@ isar_export_proxies(d)}" > +fetch_apt() { > sudo -E chroot ${BUILDCHROOT_DIR} /usr/bin/apt-get update \ > -o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \ > -o Dir::Etc::SourceParts="-" \ > @@ -72,8 +66,27 @@ do_apt_fetch() { > sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \ > sh -c 'cp /downloads/deb-src/"$1"/"$2"/* ${PP} && cd ${PP} && apt-get -y --only-source source "$2"' my_script "${DISTRO}" "${uri}" > done > +} > + > +python do_apt_fetch() { > + import glob > + import shutil > + > + src_apt = d.getVar("SRC_APT", True) > + if not src_apt: > + return 0 > > - dpkg_undo_mounts > + srcsubdir = d.getVar('S', True) > + if os.path.exists(srcsubdir): > + shutil.rmtree(srcsubdir) > + > + dpkg_do_mounts(d) > + try: > + isar_export_proxies(d) > + buildchroot = d.getVar('BUILDCHROOT_DIR', True) > + bb.build.exec_func("fetch_apt", d) > + finally: > + dpkg_undo_mounts(d) > } > > addtask apt_fetch after do_unpack before do_patch > @@ -118,25 +131,23 @@ do_prepare_build[deptask] = "do_deploy_deb" > > BUILDROOT = "${BUILDCHROOT_DIR}/${PP}" > > -dpkg_do_mounts() { > - mkdir -p ${BUILDROOT} > - sudo mount --bind ${WORKDIR} ${BUILDROOT} > - > - buildchroot_do_mounts > -} > - > -dpkg_undo_mounts() { > - i=1 > - while ! sudo umount ${BUILDROOT}; do > - sleep 0.1 > - i=`expr $i + 1` > - if [ $i -gt 100 ]; then > - bbwarn "${BUILDROOT}: Couldn't unmount, retrying..." > - i=1 > - fi > - done > - sudo rmdir ${BUILDROOT} > -} > +def dpkg_do_mounts(d): > + buildroot = d.getVar('BUILDROOT', True) > + workdir = d.getVar("WORKDIR", True) > + os.makedirs(buildroot, exist_ok=True) > + os.system('sudo mount --bind %s %s' % (workdir, buildroot)) > + bb.build.exec_func("buildchroot_do_mounts", d) > + > +def dpkg_undo_mounts(d): > + buildroot = d.getVar('BUILDROOT', True) > + for i in range(200): > + if not os.system('sudo umount %s' % buildroot): > + os.rmdir(buildroot) > + return > + if i % 100 == 0: > + bb.warn("%s: Couldn't unmount, retrying..." % buildroot) > + time.sleep(0.1) > + bb.fatal("Couldn't unmount, exiting...") > > # Placeholder for actual dpkg_runbuild() implementation > dpkg_runbuild() { > @@ -146,10 +157,12 @@ dpkg_runbuild() { > python do_dpkg_build() { > lock = bb.utils.lockfile(d.getVar("REPO_ISAR_DIR") + "/isar.lock", > shared=True) > - bb.build.exec_func("dpkg_do_mounts", d) > - bb.build.exec_func("dpkg_runbuild", d) > - bb.build.exec_func("dpkg_undo_mounts", d) > - bb.utils.unlockfile(lock) > + dpkg_do_mounts(d) > + try: > + bb.build.exec_func("dpkg_runbuild", d) > + finally: > + dpkg_undo_mounts(d) > + bb.utils.unlockfile(lock) > } > > addtask dpkg_build before do_build > @@ -193,16 +206,16 @@ python do_devshell() { > oe_lib_path = os.path.join(d.getVar('LAYERDIR_core'), 'lib') > sys.path.insert(0, oe_lib_path) > > - bb.build.exec_func('dpkg_do_mounts', d) > - > - isar_export_proxies(d) > - > - buildchroot = d.getVar('BUILDCHROOT_DIR') > - pp_pps = os.path.join(d.getVar('PP'), d.getVar('PPS')) > - termcmd = "sudo -E chroot {0} sh -c 'cd {1}; $SHELL -i'" > - oe_terminal(termcmd.format(buildchroot, pp_pps), "Isar devshell", d) > + dpkg_do_mounts(d) > + try: > + isar_export_proxies(d) > > - bb.build.exec_func('dpkg_undo_mounts', d) > + buildchroot = d.getVar('BUILDCHROOT_DIR') > + pp_pps = os.path.join(d.getVar('PP'), d.getVar('PPS')) > + termcmd = "sudo -E chroot {0} sh -c 'cd {1}; $SHELL -i'" > + oe_terminal(termcmd.format(buildchroot, pp_pps), "Isar devshell", d) > + finally: > + dpkg_undo_mounts(d) > } > > addtask devshell after do_prepare_build > diff --git a/meta/classes/dpkg-gbp.bbclass b/meta/classes/dpkg-gbp.bbclass > index d956e8c..20d2d4c 100644 > --- a/meta/classes/dpkg-gbp.bbclass > +++ b/meta/classes/dpkg-gbp.bbclass > @@ -12,12 +12,7 @@ PATCHTOOL ?= "git" > GBP_DEPENDS ?= "git-buildpackage pristine-tar" > GBP_EXTRA_OPTIONS ?= "--git-pristine-tar" > > -do_install_builddeps_append() { > - dpkg_do_mounts > - distro="${DISTRO}" > - if [ ${ISAR_CROSS_COMPILE} -eq 1 ]; then > - distro="${HOST_DISTRO}" > - fi > +builddeps_install_append() { > deb_dl_dir_import "${BUILDCHROOT_DIR}" "${distro}" > sudo -E chroot ${BUILDCHROOT_DIR} \ > apt-get install -y -o Debug::pkgProblemResolver=yes \ > @@ -26,7 +21,6 @@ do_install_builddeps_append() { > sudo -E chroot ${BUILDCHROOT_DIR} \ > apt-get install -y -o Debug::pkgProblemResolver=yes \ > --no-install-recommends ${GBP_DEPENDS} > - dpkg_undo_mounts > } > > dpkg_runbuild_prepend() { > diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass > index 4e7c2f7..29e2b89 100644 > --- a/meta/classes/dpkg.bbclass > +++ b/meta/classes/dpkg.bbclass > @@ -6,9 +6,7 @@ inherit dpkg-base > PACKAGE_ARCH ?= "${DISTRO_ARCH}" > > # Install build dependencies for package > -do_install_builddeps() { > - dpkg_do_mounts > - E="${@ isar_export_proxies(d)}" > +builddeps_install() { > distro="${DISTRO}" > if [ ${ISAR_CROSS_COMPILE} -eq 1 ]; then > distro="${HOST_DISTRO}" > @@ -19,7 +17,15 @@ do_install_builddeps() { > deb_dl_dir_export "${BUILDCHROOT_DIR}" "${distro}" > sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \ > ${PP}/${PPS} ${PACKAGE_ARCH} > - dpkg_undo_mounts > +} > + > +python do_install_builddeps() { > + dpkg_do_mounts(d) > + isar_export_proxies(d) > + try: > + bb.build.exec_func("builddeps_install", d) > + finally: > + dpkg_undo_mounts(d) > } > > addtask install_builddeps after do_prepare_build before do_dpkg_build > -- Siemens AG, T RDA IOT Corporate Competence Center Embedded Linux