[PATCH] isar-bootstrap: Fix and cleanup bind mounting

[PATCH] isar-bootstrap: Fix and cleanup bind mounting

[Jan Kiszka]

First, wrong variables were used (there is no BUILDCHROOT_DIR for
isar_bootstrap). And then we simply left the mount points active after
completing the bootstrap. The led to setup_root_file_system copying all
the mount point content over into the buildchroot or rootfs - including
base-apt when it was used.

Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
---

This is a precondition for "isar-image: umount base-apt when doing
offline build" to work properly. IOW, that patch reveal these bugs.

Possibly, this also explains the increasing mount count that was visible
in CI.

 meta/recipes-core/isar-bootstrap/isar-bootstrap.inc | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index 8ea3b08..da077d0 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -185,12 +185,8 @@ isar_bootstrap() {
     sudo -E flock "${ISAR_BOOTSTRAP_LOCK}" -c "\
         set -e
         if [ ! -e "${DEPLOY_ISAR_BOOTSTRAP}" ]; then
-            if [ -e "${ROOTFSDIR}" ]; then
-               umount -R -l "${ROOTFSDIR}/dev" || true
-               umount -l "${ROOTFSDIR}/proc" || true
-               umount -R -l "${ROOTFSDIR}/sys" || true
-               rm -rf "${ROOTFSDIR}"
-            fi
+            rm -rf "${ROOTFSDIR}"
+
             if [ ${IS_HOST} ]; then
                 ${DEBOOTSTRAP} $debootstrap_args \
                                ${@get_distro_components_argument(d, True)} \
@@ -247,17 +243,22 @@ isar_bootstrap() {
             "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
 
             # update APT
-            mount --rbind /dev ${BUILDCHROOT_DIR}/dev
-            mount --make-rslave ${BUILDCHROOT_DIR}/dev
+            mount --rbind /dev ${ROOTFSDIR}/dev
+            mount --make-rslave ${ROOTFSDIR}/dev
             mount -t proc none ${ROOTFSDIR}/proc
-            mount --rbind /sys ${BUILDCHROOT_DIR}/sys
-            mount --make-rslave ${BUILDCHROOT_DIR}/sys
+            mount --rbind /sys ${ROOTFSDIR}/sys
+            mount --make-rslave ${ROOTFSDIR}/sys
 
             export DEBIAN_FRONTEND=noninteractive
             chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
             chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
                                   -o Debug::pkgProblemResolver=yes
 
+            umount -R -l "${ROOTFSDIR}/dev"
+            umount -l "${ROOTFSDIR}/proc"
+            umount -R -l "${ROOTFSDIR}/sys"
+            umount -l "${ROOTFSDIR}/base-apt" || true
+
             # Finalize debootstrap by setting the link in deploy
             ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
         fi"
-- 
2.16.4

Re: [PATCH] isar-bootstrap: Fix and cleanup bind mounting

[Henning Schild]

Maybe that is where the leftover mounts came from, and the reverts are
not required.

Henning

Am Thu, 29 Nov 2018 17:29:17 +0100
schrieb "[ext] Jan Kiszka" <jan.kiszka@siemens.com>:

> First, wrong variables were used (there is no BUILDCHROOT_DIR for
> isar_bootstrap). And then we simply left the mount points active after
> completing the bootstrap. The led to setup_root_file_system copying
> all the mount point content over into the buildchroot or rootfs -
> including base-apt when it was used.
> 
> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
> ---
> 
> This is a precondition for "isar-image: umount base-apt when doing
> offline build" to work properly. IOW, that patch reveal these bugs.
> 
> Possibly, this also explains the increasing mount count that was
> visible in CI.
> 
>  meta/recipes-core/isar-bootstrap/isar-bootstrap.inc | 21
> +++++++++++---------- 1 file changed, 11 insertions(+), 10
> deletions(-)
> 
> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc index
> 8ea3b08..da077d0 100644 ---
> a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc +++
> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc @@ -185,12
> +185,8 @@ isar_bootstrap() { sudo -E flock "${ISAR_BOOTSTRAP_LOCK}"
> -c "\ set -e
>          if [ ! -e "${DEPLOY_ISAR_BOOTSTRAP}" ]; then
> -            if [ -e "${ROOTFSDIR}" ]; then
> -               umount -R -l "${ROOTFSDIR}/dev" || true
> -               umount -l "${ROOTFSDIR}/proc" || true
> -               umount -R -l "${ROOTFSDIR}/sys" || true
> -               rm -rf "${ROOTFSDIR}"
> -            fi
> +            rm -rf "${ROOTFSDIR}"
> +
>              if [ ${IS_HOST} ]; then
>                  ${DEBOOTSTRAP} $debootstrap_args \
>                                 ${@get_distro_components_argument(d,
> True)} \ @@ -247,17 +243,22 @@ isar_bootstrap() {
>              "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
>  
>              # update APT
> -            mount --rbind /dev ${BUILDCHROOT_DIR}/dev
> -            mount --make-rslave ${BUILDCHROOT_DIR}/dev
> +            mount --rbind /dev ${ROOTFSDIR}/dev
> +            mount --make-rslave ${ROOTFSDIR}/dev
>              mount -t proc none ${ROOTFSDIR}/proc
> -            mount --rbind /sys ${BUILDCHROOT_DIR}/sys
> -            mount --make-rslave ${BUILDCHROOT_DIR}/sys
> +            mount --rbind /sys ${ROOTFSDIR}/sys
> +            mount --make-rslave ${ROOTFSDIR}/sys
>  
>              export DEBIAN_FRONTEND=noninteractive
>              chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
>              chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
>                                    -o Debug::pkgProblemResolver=yes
>  
> +            umount -R -l "${ROOTFSDIR}/dev"
> +            umount -l "${ROOTFSDIR}/proc"
> +            umount -R -l "${ROOTFSDIR}/sys"
> +            umount -l "${ROOTFSDIR}/base-apt" || true
> +
>              # Finalize debootstrap by setting the link in deploy
>              ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
>          fi"

Re: [PATCH] isar-bootstrap: Fix and cleanup bind mounting

[Maxim Yu. Osipov]

Hi Jan,

I've just tried this patch (on the 'next' with reverted patch d40a9ac0) 
and ran "fast" CI

isar$mount | wc -l
34

isar$./scripts/ci_build.sh -q -f

CI script hung on CI stage when dpkg-base is modified
causing rebuilding recipes based on dpkg-base.

The mount reports less (!) mount points than before launching the script.

mount | wc -l
31

Maxim.

On 11/29/18 9:39 PM, Henning Schild wrote:
> Maybe that is where the leftover mounts came from, and the reverts are
> not required.
> 
> Henning
> 
> Am Thu, 29 Nov 2018 17:29:17 +0100
> schrieb "[ext] Jan Kiszka" <jan.kiszka@siemens.com>:
> 
>> First, wrong variables were used (there is no BUILDCHROOT_DIR for
>> isar_bootstrap). And then we simply left the mount points active after
>> completing the bootstrap. The led to setup_root_file_system copying
>> all the mount point content over into the buildchroot or rootfs -
>> including base-apt when it was used.
>>
>> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
>> ---
>>
>> This is a precondition for "isar-image: umount base-apt when doing
>> offline build" to work properly. IOW, that patch reveal these bugs.
>>
>> Possibly, this also explains the increasing mount count that was
>> visible in CI.
>>
>>   meta/recipes-core/isar-bootstrap/isar-bootstrap.inc | 21
>> +++++++++++---------- 1 file changed, 11 insertions(+), 10
>> deletions(-)
>>
>> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc index
>> 8ea3b08..da077d0 100644 ---
>> a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc +++
>> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc @@ -185,12
>> +185,8 @@ isar_bootstrap() { sudo -E flock "${ISAR_BOOTSTRAP_LOCK}"
>> -c "\ set -e
>>           if [ ! -e "${DEPLOY_ISAR_BOOTSTRAP}" ]; then
>> -            if [ -e "${ROOTFSDIR}" ]; then
>> -               umount -R -l "${ROOTFSDIR}/dev" || true
>> -               umount -l "${ROOTFSDIR}/proc" || true
>> -               umount -R -l "${ROOTFSDIR}/sys" || true
>> -               rm -rf "${ROOTFSDIR}"
>> -            fi
>> +            rm -rf "${ROOTFSDIR}"
>> +
>>               if [ ${IS_HOST} ]; then
>>                   ${DEBOOTSTRAP} $debootstrap_args \
>>                                  ${@get_distro_components_argument(d,
>> True)} \ @@ -247,17 +243,22 @@ isar_bootstrap() {
>>               "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
>>   
>>               # update APT
>> -            mount --rbind /dev ${BUILDCHROOT_DIR}/dev
>> -            mount --make-rslave ${BUILDCHROOT_DIR}/dev
>> +            mount --rbind /dev ${ROOTFSDIR}/dev
>> +            mount --make-rslave ${ROOTFSDIR}/dev
>>               mount -t proc none ${ROOTFSDIR}/proc
>> -            mount --rbind /sys ${BUILDCHROOT_DIR}/sys
>> -            mount --make-rslave ${BUILDCHROOT_DIR}/sys
>> +            mount --rbind /sys ${ROOTFSDIR}/sys
>> +            mount --make-rslave ${ROOTFSDIR}/sys
>>   
>>               export DEBIAN_FRONTEND=noninteractive
>>               chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
>>               chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
>>                                     -o Debug::pkgProblemResolver=yes
>>   
>> +            umount -R -l "${ROOTFSDIR}/dev"
>> +            umount -l "${ROOTFSDIR}/proc"
>> +            umount -R -l "${ROOTFSDIR}/sys"
>> +            umount -l "${ROOTFSDIR}/base-apt" || true
>> +
>>               # Finalize debootstrap by setting the link in deploy
>>               ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
>>           fi"
> 


-- 
Maxim Osipov
ilbers GmbH
Maria-Merian-Str. 8
85521 Ottobrunn
Germany
+49 (151) 6517 6917
mosipov@ilbers.de
http://ilbers.de/
Commercial register Munich, HRB 214197
General Manager: Baurzhan Ismagulov

Re: [PATCH] isar-bootstrap: Fix and cleanup bind mounting

[Henning Schild]

Am Fri, 30 Nov 2018 12:20:29 +0300
schrieb "Maxim Yu. Osipov" <mosipov@ilbers.de>:

> Hi Jan,
> 
> I've just tried this patch (on the 'next' with reverted patch
> d40a9ac0) and ran "fast" CI
> 
> isar$mount | wc -l
> 34
> 
> isar$./scripts/ci_build.sh -q -f
> 
> CI script hung on CI stage when dpkg-base is modified
> causing rebuilding recipes based on dpkg-base.
> 
> The mount reports less (!) mount points than before launching the
> script.
> 
> mount | wc -l
> 31

Maybe you can share the output of mount, i would blame the "umount
--lazy". I am not sure why this was introduces, would be good if we
could get rid of it.

Henning

> Maxim.
> 
> On 11/29/18 9:39 PM, Henning Schild wrote:
> > Maybe that is where the leftover mounts came from, and the reverts
> > are not required.
> > 
> > Henning
> > 
> > Am Thu, 29 Nov 2018 17:29:17 +0100
> > schrieb "[ext] Jan Kiszka" <jan.kiszka@siemens.com>:
> >   
> >> First, wrong variables were used (there is no BUILDCHROOT_DIR for
> >> isar_bootstrap). And then we simply left the mount points active
> >> after completing the bootstrap. The led to setup_root_file_system
> >> copying all the mount point content over into the buildchroot or
> >> rootfs - including base-apt when it was used.
> >>
> >> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
> >> ---
> >>
> >> This is a precondition for "isar-image: umount base-apt when doing
> >> offline build" to work properly. IOW, that patch reveal these bugs.
> >>
> >> Possibly, this also explains the increasing mount count that was
> >> visible in CI.
> >>
> >>   meta/recipes-core/isar-bootstrap/isar-bootstrap.inc | 21
> >> +++++++++++---------- 1 file changed, 11 insertions(+), 10
> >> deletions(-)
> >>
> >> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> >> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc index
> >> 8ea3b08..da077d0 100644 ---
> >> a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc +++
> >> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc @@ -185,12
> >> +185,8 @@ isar_bootstrap() { sudo -E flock "${ISAR_BOOTSTRAP_LOCK}"
> >> -c "\ set -e
> >>           if [ ! -e "${DEPLOY_ISAR_BOOTSTRAP}" ]; then
> >> -            if [ -e "${ROOTFSDIR}" ]; then
> >> -               umount -R -l "${ROOTFSDIR}/dev" || true
> >> -               umount -l "${ROOTFSDIR}/proc" || true
> >> -               umount -R -l "${ROOTFSDIR}/sys" || true
> >> -               rm -rf "${ROOTFSDIR}"
> >> -            fi
> >> +            rm -rf "${ROOTFSDIR}"
> >> +
> >>               if [ ${IS_HOST} ]; then
> >>                   ${DEBOOTSTRAP} $debootstrap_args \
> >>                                  ${@get_distro_components_argument(d,
> >> True)} \ @@ -247,17 +243,22 @@ isar_bootstrap() {
> >>               "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
> >>   
> >>               # update APT
> >> -            mount --rbind /dev ${BUILDCHROOT_DIR}/dev
> >> -            mount --make-rslave ${BUILDCHROOT_DIR}/dev
> >> +            mount --rbind /dev ${ROOTFSDIR}/dev
> >> +            mount --make-rslave ${ROOTFSDIR}/dev
> >>               mount -t proc none ${ROOTFSDIR}/proc
> >> -            mount --rbind /sys ${BUILDCHROOT_DIR}/sys
> >> -            mount --make-rslave ${BUILDCHROOT_DIR}/sys
> >> +            mount --rbind /sys ${ROOTFSDIR}/sys
> >> +            mount --make-rslave ${ROOTFSDIR}/sys
> >>   
> >>               export DEBIAN_FRONTEND=noninteractive
> >>               chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
> >>               chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade
> >> -y \ -o Debug::pkgProblemResolver=yes
> >>   
> >> +            umount -R -l "${ROOTFSDIR}/dev"
> >> +            umount -l "${ROOTFSDIR}/proc"
> >> +            umount -R -l "${ROOTFSDIR}/sys"
> >> +            umount -l "${ROOTFSDIR}/base-apt" || true
> >> +
> >>               # Finalize debootstrap by setting the link in deploy
> >>               ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
> >>           fi"  
> >   
> 
> 

Re: [PATCH] isar-bootstrap: Fix and cleanup bind mounting

[Jan Kiszka]

On 30.11.18 10:36, Henning Schild wrote:
> Am Fri, 30 Nov 2018 12:20:29 +0300
> schrieb "Maxim Yu. Osipov" <mosipov@ilbers.de>:
> 
>> Hi Jan,
>>
>> I've just tried this patch (on the 'next' with reverted patch
>> d40a9ac0) and ran "fast" CI
>>
>> isar$mount | wc -l
>> 34
>>
>> isar$./scripts/ci_build.sh -q -f
>>
>> CI script hung on CI stage when dpkg-base is modified
>> causing rebuilding recipes based on dpkg-base.
>>
>> The mount reports less (!) mount points than before launching the
>> script.
>>
>> mount | wc -l
>> 31
> 
> Maybe you can share the output of mount, i would blame the "umount
> --lazy". I am not sure why this was introduces, would be good if we
> could get rid of it.
> 

umount -l was already used by Claudius in April, when refactoring the bootstrap. 
But you may still be right.

Jan

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

Re: [PATCH] isar-bootstrap: Fix and cleanup bind mounting

[Jan Kiszka]

On 30.11.18 10:20, Maxim Yu. Osipov wrote:
> Hi Jan,
> 
> I've just tried this patch (on the 'next' with reverted patch d40a9ac0) and ran 
> "fast" CI
> 
> isar$mount | wc -l
> 34
> 
> isar$./scripts/ci_build.sh -q -f
> 
> CI script hung on CI stage when dpkg-base is modified
> causing rebuilding recipes based on dpkg-base.
> 
> The mount reports less (!) mount points than before launching the script.
> 
> mount | wc -l
> 31

I will try to reproduce locally. So far, none of the test I made with downstream 
layers reported issues. But we may have more hidden bugs...

Jan

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

Re: [PATCH] isar-bootstrap: Fix and cleanup bind mounting

[Jan Kiszka]

On 30.11.18 10:38, Jan Kiszka wrote:
> On 30.11.18 10:20, Maxim Yu. Osipov wrote:
>> Hi Jan,
>>
>> I've just tried this patch (on the 'next' with reverted patch d40a9ac0) and 
>> ran "fast" CI
>>
>> isar$mount | wc -l
>> 34
>>
>> isar$./scripts/ci_build.sh -q -f
>>
>> CI script hung on CI stage when dpkg-base is modified
>> causing rebuilding recipes based on dpkg-base.
>>
>> The mount reports less (!) mount points than before launching the script.
>>
>> mount | wc -l
>> 31
> 
> I will try to reproduce locally. So far, none of the test I made with downstream 
> layers reported issues. But we may have more hidden bugs...

Just finished here, but it doesn't trigger. Also, no suspicious mounts where 
left behind (but I forgot the measure their number).

Jan

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

Re: [PATCH] isar-bootstrap: Fix and cleanup bind mounting

[Jan Kiszka]

On 30.11.18 11:19, Jan Kiszka wrote:
> On 30.11.18 10:38, Jan Kiszka wrote:
>> On 30.11.18 10:20, Maxim Yu. Osipov wrote:
>>> Hi Jan,
>>>
>>> I've just tried this patch (on the 'next' with reverted patch d40a9ac0) and 
>>> ran "fast" CI
>>>
>>> isar$mount | wc -l
>>> 34
>>>
>>> isar$./scripts/ci_build.sh -q -f
>>>
>>> CI script hung on CI stage when dpkg-base is modified
>>> causing rebuilding recipes based on dpkg-base.
>>>
>>> The mount reports less (!) mount points than before launching the script.
>>>
>>> mount | wc -l
>>> 31
>>
>> I will try to reproduce locally. So far, none of the test I made with 
>> downstream layers reported issues. But we may have more hidden bugs...
> 
> Just finished here, but it doesn't trigger. Also, no suspicious mounts where 
> left behind (but I forgot the measure their number).
> 

Re-ran, this time counting mounts: 27 before and after, no hangs.

FWIW, this is my queue used for that:
https://github.com/siemens/isar/commits/jan/queue

Jan

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

Re: [PATCH] isar-bootstrap: Fix and cleanup bind mounting

[Jan Kiszka]

On 30.11.18 10:20, Maxim Yu. Osipov wrote:
> Hi Jan,
> 
> I've just tried this patch (on the 'next' with reverted patch d40a9ac0) and ran 
> "fast" CI
> 
> isar$mount | wc -l
> 34
> 
> isar$./scripts/ci_build.sh -q -f
> 
> CI script hung on CI stage when dpkg-base is modified
> causing rebuilding recipes based on dpkg-base.
> 
> The mount reports less (!) mount points than before launching the script.
> 
> mount | wc -l
> 31

Any news on what's different on your side? Where exactly does your build hang? 
Was your CI environment in a clean state when running this test? Before the 
comment lots of things leaked.

We still do not see problems with this bug fix, neither locally nor in CI (which 
is ci_build-based now, too). And it's actually a precondition to make 
"isar-image: umount base-apt when doing offline build" work as expected.

Thanks,
Jan

> 
> Maxim.
> 
> On 11/29/18 9:39 PM, Henning Schild wrote:
>> Maybe that is where the leftover mounts came from, and the reverts are
>> not required.
>>
>> Henning
>>
>> Am Thu, 29 Nov 2018 17:29:17 +0100
>> schrieb "[ext] Jan Kiszka" <jan.kiszka@siemens.com>:
>>
>>> First, wrong variables were used (there is no BUILDCHROOT_DIR for
>>> isar_bootstrap). And then we simply left the mount points active after
>>> completing the bootstrap. The led to setup_root_file_system copying
>>> all the mount point content over into the buildchroot or rootfs -
>>> including base-apt when it was used.
>>>
>>> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
>>> ---
>>>
>>> This is a precondition for "isar-image: umount base-apt when doing
>>> offline build" to work properly. IOW, that patch reveal these bugs.
>>>
>>> Possibly, this also explains the increasing mount count that was
>>> visible in CI.
>>>
>>>   meta/recipes-core/isar-bootstrap/isar-bootstrap.inc | 21
>>> +++++++++++---------- 1 file changed, 11 insertions(+), 10
>>> deletions(-)
>>>
>>> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>>> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc index
>>> 8ea3b08..da077d0 100644 ---
>>> a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc +++
>>> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc @@ -185,12
>>> +185,8 @@ isar_bootstrap() { sudo -E flock "${ISAR_BOOTSTRAP_LOCK}"
>>> -c "\ set -e
>>>           if [ ! -e "${DEPLOY_ISAR_BOOTSTRAP}" ]; then
>>> -            if [ -e "${ROOTFSDIR}" ]; then
>>> -               umount -R -l "${ROOTFSDIR}/dev" || true
>>> -               umount -l "${ROOTFSDIR}/proc" || true
>>> -               umount -R -l "${ROOTFSDIR}/sys" || true
>>> -               rm -rf "${ROOTFSDIR}"
>>> -            fi
>>> +            rm -rf "${ROOTFSDIR}"
>>> +
>>>               if [ ${IS_HOST} ]; then
>>>                   ${DEBOOTSTRAP} $debootstrap_args \
>>>                                  ${@get_distro_components_argument(d,
>>> True)} \ @@ -247,17 +243,22 @@ isar_bootstrap() {
>>>               "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
>>>               # update APT
>>> -            mount --rbind /dev ${BUILDCHROOT_DIR}/dev
>>> -            mount --make-rslave ${BUILDCHROOT_DIR}/dev
>>> +            mount --rbind /dev ${ROOTFSDIR}/dev
>>> +            mount --make-rslave ${ROOTFSDIR}/dev
>>>               mount -t proc none ${ROOTFSDIR}/proc
>>> -            mount --rbind /sys ${BUILDCHROOT_DIR}/sys
>>> -            mount --make-rslave ${BUILDCHROOT_DIR}/sys
>>> +            mount --rbind /sys ${ROOTFSDIR}/sys
>>> +            mount --make-rslave ${ROOTFSDIR}/sys
>>>               export DEBIAN_FRONTEND=noninteractive
>>>               chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
>>>               chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
>>>                                     -o Debug::pkgProblemResolver=yes
>>> +            umount -R -l "${ROOTFSDIR}/dev"
>>> +            umount -l "${ROOTFSDIR}/proc"
>>> +            umount -R -l "${ROOTFSDIR}/sys"
>>> +            umount -l "${ROOTFSDIR}/base-apt" || true
>>> +
>>>               # Finalize debootstrap by setting the link in deploy
>>>               ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
>>>           fi"
>>
> 
> 

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

RE: [PATCH] isar-bootstrap: Fix and cleanup bind mounting

[Hombourger, Cedric]

FWIW, I was also able to run the ci tests within my modest VM and did not observe any failures
The changes from Jan are fixing genuine issues and should probably be merged (assuming they pass code reviews)

Cedric

-----Original Message-----
From: Jan Kiszka [mailto:jan.kiszka@siemens.com] 
Sent: Monday, December 3, 2018 3:59 PM
To: Maxim Yu. Osipov <mosipov@ilbers.de>
Cc: Henning Schild <henning.schild@siemens.com>; isar-users <isar-users@googlegroups.com>; Hombourger, Cedric <Cedric_Hombourger@mentor.com>
Subject: Re: [PATCH] isar-bootstrap: Fix and cleanup bind mounting

On 30.11.18 10:20, Maxim Yu. Osipov wrote:
> Hi Jan,
> 
> I've just tried this patch (on the 'next' with reverted patch 
> d40a9ac0) and ran "fast" CI
> 
> isar$mount | wc -l
> 34
> 
> isar$./scripts/ci_build.sh -q -f
> 
> CI script hung on CI stage when dpkg-base is modified causing 
> rebuilding recipes based on dpkg-base.
> 
> The mount reports less (!) mount points than before launching the script.
> 
> mount | wc -l
> 31

Any news on what's different on your side? Where exactly does your build hang? 
Was your CI environment in a clean state when running this test? Before the comment lots of things leaked.

We still do not see problems with this bug fix, neither locally nor in CI (which is ci_build-based now, too). And it's actually a precondition to make
"isar-image: umount base-apt when doing offline build" work as expected.

Thanks,
Jan

> 
> Maxim.
> 
> On 11/29/18 9:39 PM, Henning Schild wrote:
>> Maybe that is where the leftover mounts came from, and the reverts 
>> are not required.
>>
>> Henning
>>
>> Am Thu, 29 Nov 2018 17:29:17 +0100
>> schrieb "[ext] Jan Kiszka" <jan.kiszka@siemens.com>:
>>
>>> First, wrong variables were used (there is no BUILDCHROOT_DIR for 
>>> isar_bootstrap). And then we simply left the mount points active 
>>> after completing the bootstrap. The led to setup_root_file_system 
>>> copying all the mount point content over into the buildchroot or 
>>> rootfs - including base-apt when it was used.
>>>
>>> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
>>> ---
>>>
>>> This is a precondition for "isar-image: umount base-apt when doing 
>>> offline build" to work properly. IOW, that patch reveal these bugs.
>>>
>>> Possibly, this also explains the increasing mount count that was 
>>> visible in CI.
>>>
>>>   meta/recipes-core/isar-bootstrap/isar-bootstrap.inc | 21
>>> +++++++++++---------- 1 file changed, 11 insertions(+), 10
>>> deletions(-)
>>>
>>> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>>> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc index
>>> 8ea3b08..da077d0 100644 ---
>>> a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc +++ 
>>> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc @@ -185,12
>>> +185,8 @@ isar_bootstrap() { sudo -E flock "${ISAR_BOOTSTRAP_LOCK}"
>>> -c "\ set -e
>>>           if [ ! -e "${DEPLOY_ISAR_BOOTSTRAP}" ]; then
>>> -            if [ -e "${ROOTFSDIR}" ]; then
>>> -               umount -R -l "${ROOTFSDIR}/dev" || true
>>> -               umount -l "${ROOTFSDIR}/proc" || true
>>> -               umount -R -l "${ROOTFSDIR}/sys" || true
>>> -               rm -rf "${ROOTFSDIR}"
>>> -            fi
>>> +            rm -rf "${ROOTFSDIR}"
>>> +
>>>               if [ ${IS_HOST} ]; then
>>>                   ${DEBOOTSTRAP} $debootstrap_args \
>>>                                  
>>> ${@get_distro_components_argument(d,
>>> True)} \ @@ -247,17 +243,22 @@ isar_bootstrap() {
>>>               "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
>>>               # update APT
>>> -            mount --rbind /dev ${BUILDCHROOT_DIR}/dev
>>> -            mount --make-rslave ${BUILDCHROOT_DIR}/dev
>>> +            mount --rbind /dev ${ROOTFSDIR}/dev
>>> +            mount --make-rslave ${ROOTFSDIR}/dev
>>>               mount -t proc none ${ROOTFSDIR}/proc
>>> -            mount --rbind /sys ${BUILDCHROOT_DIR}/sys
>>> -            mount --make-rslave ${BUILDCHROOT_DIR}/sys
>>> +            mount --rbind /sys ${ROOTFSDIR}/sys
>>> +            mount --make-rslave ${ROOTFSDIR}/sys
>>>               export DEBIAN_FRONTEND=noninteractive
>>>               chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
>>>               chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y 
>>> \
>>>                                     -o Debug::pkgProblemResolver=yes
>>> +            umount -R -l "${ROOTFSDIR}/dev"
>>> +            umount -l "${ROOTFSDIR}/proc"
>>> +            umount -R -l "${ROOTFSDIR}/sys"
>>> +            umount -l "${ROOTFSDIR}/base-apt" || true
>>> +
>>>               # Finalize debootstrap by setting the link in deploy
>>>               ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
>>>           fi"
>>
> 
> 

--
Siemens AG, Corporate Technology, CT RDA IOT SES-DE Corporate Competence Center Embedded Linux

Re: [PATCH] isar-bootstrap: Fix and cleanup bind mounting

[Maxim Yu. Osipov]

[-- Attachment #1: Type: text/plain, Size: 6468 bytes --]

On 12/3/18 3:59 PM, Jan Kiszka wrote:
> On 30.11.18 10:20, Maxim Yu. Osipov wrote:
>> Hi Jan,
>>
>> I've just tried this patch (on the 'next' with reverted patch 
>> d40a9ac0) and ran "fast" CI
>>
>> isar$mount | wc -l
>> 34
>>
>> isar$./scripts/ci_build.sh -q -f
>>
>> CI script hung on CI stage when dpkg-base is modified
>> causing rebuilding recipes based on dpkg-base.
>>
>> The mount reports less (!) mount points than before launching the script.
>>
>> mount | wc -l
>> 31
> 
> Any news on what's different on your side? Where exactly does your build 
> hang? Was your CI environment in a clean state when running this test? 
> Before the comment lots of things leaked.


On my stretch laptop (i7-6820HQ CPU @ 2.70GHz (8 cores) with SSD)  the 
reported problem is reproducible (I rerun 'ci_build.sh -q -f' several 
times in clean state) it hung and with the less mount points (the mount 
points before and after running are attached).

The strange thing that I observe two bitbake processes:

myo      26373  0.0  0.3 153116 29732 pts/0    Sl+  12:31   0:01 python3 
/home/myo/work/isar/src/trunk/isar/bitbake/bin/bitbake 
multiconfig:qemuarm-stretch:isar-image-base 
multiconfig:qemuarm64-stretch:isar-image-base 
multiconfig:qemuamd64-stretch:isar-image-base

myo      26379  2.5  0.6 328476 50028 ?        Sl   12:31   0:40 python3 
/home/myo/work/isar/src/trunk/isar/bitbake/bin/bitbake 
multiconfig:qemuarm-stretch:isar-image-base 
multiconfig:qemuarm64-stretch:isar-image-base 
multiconfig:qemuamd64-stretch:isar-image-base


When I turned on verbosity (called ci_build.sh without '-q' switch)
'scripts/ci_build -f' succeeded.

Build also fails on isar-build.org
http://isar-build.org:8080/job/isar_mosipov_next/98/console:
it complains  on
'rm: cannot remove 
'/workspace/build/isar_mosipov_next/98/build/tmp/work/debian-stretch-armhf/isar-bootstrap-target/rootfs/dev': 
Device or resource busy'

Note that ci-build.sh on isar-build.org is also called with '-q' switch.


Questions:

1) Did you launch ci_build with '-q' option?

2) Did you inspect mount points before and after launching ci_build? Are 
they identical?

3) Do you run this in docker?


Maxim.


> We still do not see problems with this bug fix, neither locally nor in 
> CI (which is ci_build-based now, too). And it's actually a precondition 
> to make "isar-image: umount base-apt when doing offline build" work as 
> expected.
> 
> Thanks,
> Jan
> 
>>
>> Maxim.
>>
>> On 11/29/18 9:39 PM, Henning Schild wrote:
>>> Maybe that is where the leftover mounts came from, and the reverts are
>>> not required.
>>>
>>> Henning
>>>
>>> Am Thu, 29 Nov 2018 17:29:17 +0100
>>> schrieb "[ext] Jan Kiszka" <jan.kiszka@siemens.com>:
>>>
>>>> First, wrong variables were used (there is no BUILDCHROOT_DIR for
>>>> isar_bootstrap). And then we simply left the mount points active after
>>>> completing the bootstrap. The led to setup_root_file_system copying
>>>> all the mount point content over into the buildchroot or rootfs -
>>>> including base-apt when it was used.
>>>>
>>>> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
>>>> ---
>>>>
>>>> This is a precondition for "isar-image: umount base-apt when doing
>>>> offline build" to work properly. IOW, that patch reveal these bugs.
>>>>
>>>> Possibly, this also explains the increasing mount count that was
>>>> visible in CI.
>>>>
>>>>   meta/recipes-core/isar-bootstrap/isar-bootstrap.inc | 21
>>>> +++++++++++---------- 1 file changed, 11 insertions(+), 10
>>>> deletions(-)
>>>>
>>>> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>>>> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc index
>>>> 8ea3b08..da077d0 100644 ---
>>>> a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc +++
>>>> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc @@ -185,12
>>>> +185,8 @@ isar_bootstrap() { sudo -E flock "${ISAR_BOOTSTRAP_LOCK}"
>>>> -c "\ set -e
>>>>           if [ ! -e "${DEPLOY_ISAR_BOOTSTRAP}" ]; then
>>>> -            if [ -e "${ROOTFSDIR}" ]; then
>>>> -               umount -R -l "${ROOTFSDIR}/dev" || true
>>>> -               umount -l "${ROOTFSDIR}/proc" || true
>>>> -               umount -R -l "${ROOTFSDIR}/sys" || true
>>>> -               rm -rf "${ROOTFSDIR}"
>>>> -            fi
>>>> +            rm -rf "${ROOTFSDIR}"
>>>> +
>>>>               if [ ${IS_HOST} ]; then
>>>>                   ${DEBOOTSTRAP} $debootstrap_args \
>>>>                                  ${@get_distro_components_argument(d,
>>>> True)} \ @@ -247,17 +243,22 @@ isar_bootstrap() {
>>>>               "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
>>>>               # update APT
>>>> -            mount --rbind /dev ${BUILDCHROOT_DIR}/dev
>>>> -            mount --make-rslave ${BUILDCHROOT_DIR}/dev
>>>> +            mount --rbind /dev ${ROOTFSDIR}/dev
>>>> +            mount --make-rslave ${ROOTFSDIR}/dev
>>>>               mount -t proc none ${ROOTFSDIR}/proc
>>>> -            mount --rbind /sys ${BUILDCHROOT_DIR}/sys
>>>> -            mount --make-rslave ${BUILDCHROOT_DIR}/sys
>>>> +            mount --rbind /sys ${ROOTFSDIR}/sys
>>>> +            mount --make-rslave ${ROOTFSDIR}/sys
>>>>               export DEBIAN_FRONTEND=noninteractive
>>>>               chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
>>>>               chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
>>>>                                     -o Debug::pkgProblemResolver=yes
>>>> +            umount -R -l "${ROOTFSDIR}/dev"
>>>> +            umount -l "${ROOTFSDIR}/proc"
>>>> +            umount -R -l "${ROOTFSDIR}/sys"
>>>> +            umount -l "${ROOTFSDIR}/base-apt" || true
>>>> +
>>>>               # Finalize debootstrap by setting the link in deploy
>>>>               ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
>>>>           fi"
>>>
>>
>>
> 


-- 
Maxim Osipov
ilbers GmbH
Maria-Merian-Str. 8
85521 Ottobrunn
Germany
+49 (151) 6517 6917
mosipov@ilbers.de
http://ilbers.de/
Commercial register Munich, HRB 214197
General Manager: Baurzhan Ismagulov

[-- Attachment #2: mount_after.txt --]
[-- Type: text/plain, Size: 5063 bytes --]

sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime)
proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)
udev on /dev type devtmpfs (rw,nosuid,relatime,size=4015904k,nr_inodes=1003976,mode=755)
devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000)
tmpfs on /run type tmpfs (rw,nosuid,noexec,relatime,size=805840k,mode=755)
/dev/sda3 on / type ext4 (rw,relatime,errors=remount-ro,data=ordered)
tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev)
tmpfs on /run/lock type tmpfs (rw,nosuid,nodev,noexec,relatime,size=5120k)
tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755)
cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd)
systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=38,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=235)
/dev/sda5 on /home type ext4 (rw,relatime,data=ordered)
/dev/sda1 on /boot/efi type vfat (rw,relatime,fmask=0002,dmask=0002,allow_utime=0020,codepage=437,iocharset=ascii,shortname=mixed,quiet,utf8,errors=remount-ro)
binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime)
tmpfs on /run/user/114 type tmpfs (rw,nosuid,nodev,relatime,size=805836k,mode=700,uid=114,gid=116)
gvfsd-fuse on /run/user/114/gvfs type fuse.gvfsd-fuse (rw,nosuid,nodev,relatime,user_id=114,group_id=116)
tmpfs on /run/user/1000 type tmpfs (rw,nosuid,nodev,relatime,size=805836k,mode=700,uid=1000,gid=1000)
gvfsd-fuse on /run/user/1000/gvfs type fuse.gvfsd-fuse (rw,nosuid,nodev,relatime,user_id=1000,group_id=1000)
/dev/sda5 on /home/myo/work/isar/src/trunk/isar/build/tmp/work/debian-stretch-amd64/buildchroot-target/rootfs/home/myo/work/isar/src/trunk/isar/build/tmp type ext4 (rw,relatime,data=ordered)
udev on /home/myo/work/isar/src/trunk/isar/build/tmp/work/debian-stretch-amd64/buildchroot-target/rootfs/home/myo/work/isar/src/trunk/isar/build/tmp/work/debian-stretch-arm64/isar-image-base/rootfs/dev type devtmpfs (rw,nosuid,relatime,size=4015904k,nr_inodes=1003976,mode=755)
devpts on /home/myo/work/isar/src/trunk/isar/build/tmp/work/debian-stretch-amd64/buildchroot-target/rootfs/home/myo/work/isar/src/trunk/isar/build/tmp/work/debian-stretch-arm64/isar-image-base/rootfs/dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000)
tmpfs on /home/myo/work/isar/src/trunk/isar/build/tmp/work/debian-stretch-amd64/buildchroot-target/rootfs/home/myo/work/isar/src/trunk/isar/build/tmp/work/debian-stretch-arm64/isar-image-base/rootfs/dev/shm type tmpfs (rw,nosuid,nodev)
sysfs on /home/myo/work/isar/src/trunk/isar/build/tmp/work/debian-stretch-amd64/buildchroot-target/rootfs/home/myo/work/isar/src/trunk/isar/build/tmp/work/debian-stretch-arm64/isar-image-base/rootfs/sys type sysfs (rw,nosuid,nodev,noexec,relatime)
tmpfs on /home/myo/work/isar/src/trunk/isar/build/tmp/work/debian-stretch-amd64/buildchroot-target/rootfs/home/myo/work/isar/src/trunk/isar/build/tmp/work/debian-stretch-arm64/isar-image-base/rootfs/sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755)
cgroup on /home/myo/work/isar/src/trunk/isar/build/tmp/work/debian-stretch-amd64/buildchroot-target/rootfs/home/myo/work/isar/src/trunk/isar/build/tmp/work/debian-stretch-arm64/isar-image-base/rootfs/sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd)
udev on /home/myo/work/isar/src/trunk/isar/build/tmp/work/debian-stretch-amd64/buildchroot-target/rootfs/home/myo/work/isar/src/trunk/isar/build/tmp/work/debian-stretch-armhf/isar-image-base/rootfs/dev type devtmpfs (rw,nosuid,relatime,size=4015904k,nr_inodes=1003976,mode=755)
devpts on /home/myo/work/isar/src/trunk/isar/build/tmp/work/debian-stretch-amd64/buildchroot-target/rootfs/home/myo/work/isar/src/trunk/isar/build/tmp/work/debian-stretch-armhf/isar-image-base/rootfs/dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000)
tmpfs on /home/myo/work/isar/src/trunk/isar/build/tmp/work/debian-stretch-amd64/buildchroot-target/rootfs/home/myo/work/isar/src/trunk/isar/build/tmp/work/debian-stretch-armhf/isar-image-base/rootfs/dev/shm type tmpfs (rw,nosuid,nodev)
sysfs on /home/myo/work/isar/src/trunk/isar/build/tmp/work/debian-stretch-amd64/buildchroot-target/rootfs/home/myo/work/isar/src/trunk/isar/build/tmp/work/debian-stretch-armhf/isar-image-base/rootfs/sys type sysfs (rw,nosuid,nodev,noexec,relatime)
tmpfs on /home/myo/work/isar/src/trunk/isar/build/tmp/work/debian-stretch-amd64/buildchroot-target/rootfs/home/myo/work/isar/src/trunk/isar/build/tmp/work/debian-stretch-armhf/isar-image-base/rootfs/sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755)
cgroup on /home/myo/work/isar/src/trunk/isar/build/tmp/work/debian-stretch-amd64/buildchroot-target/rootfs/home/myo/work/isar/src/trunk/isar/build/tmp/work/debian-stretch-armhf/isar-image-base/rootfs/sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd)

[-- Attachment #3: mount_before.txt --]
[-- Type: text/plain, Size: 2875 bytes --]

sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime)
proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)
udev on /dev type devtmpfs (rw,nosuid,relatime,size=4015904k,nr_inodes=1003976,mode=755)
devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000)
tmpfs on /run type tmpfs (rw,nosuid,noexec,relatime,size=805840k,mode=755)
/dev/sda3 on / type ext4 (rw,relatime,errors=remount-ro,data=ordered)
securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime)
tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev)
tmpfs on /run/lock type tmpfs (rw,nosuid,nodev,noexec,relatime,size=5120k)
tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755)
cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd)
pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime)
efivarfs on /sys/firmware/efi/efivars type efivarfs (rw,nosuid,nodev,noexec,relatime)
cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset)
cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpu,cpuacct)
cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices)
cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer)
cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory)
cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio)
cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_cls,net_prio)
cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event)
cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids)
systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=38,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=235)
mqueue on /dev/mqueue type mqueue (rw,relatime)
debugfs on /sys/kernel/debug type debugfs (rw,relatime)
hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime)
/dev/sda5 on /home type ext4 (rw,relatime,data=ordered)
/dev/sda1 on /boot/efi type vfat (rw,relatime,fmask=0002,dmask=0002,allow_utime=0020,codepage=437,iocharset=ascii,shortname=mixed,quiet,utf8,errors=remount-ro)
binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime)
tmpfs on /run/user/114 type tmpfs (rw,nosuid,nodev,relatime,size=805836k,mode=700,uid=114,gid=116)
gvfsd-fuse on /run/user/114/gvfs type fuse.gvfsd-fuse (rw,nosuid,nodev,relatime,user_id=114,group_id=116)
fusectl on /sys/fs/fuse/connections type fusectl (rw,relatime)
tmpfs on /run/user/1000 type tmpfs (rw,nosuid,nodev,relatime,size=805836k,mode=700,uid=1000,gid=1000)
gvfsd-fuse on /run/user/1000/gvfs type fuse.gvfsd-fuse (rw,nosuid,nodev,relatime,user_id=1000,group_id=1000)

Re: [PATCH] isar-bootstrap: Fix and cleanup bind mounting

[Jan Kiszka]

On 04.12.18 11:49, Maxim Yu. Osipov wrote:
> On 12/3/18 3:59 PM, Jan Kiszka wrote:
>> On 30.11.18 10:20, Maxim Yu. Osipov wrote:
>>> Hi Jan,
>>>
>>> I've just tried this patch (on the 'next' with reverted patch d40a9ac0) and 
>>> ran "fast" CI
>>>
>>> isar$mount | wc -l
>>> 34
>>>
>>> isar$./scripts/ci_build.sh -q -f
>>>
>>> CI script hung on CI stage when dpkg-base is modified
>>> causing rebuilding recipes based on dpkg-base.
>>>
>>> The mount reports less (!) mount points than before launching the script.
>>>
>>> mount | wc -l
>>> 31
>>
>> Any news on what's different on your side? Where exactly does your build hang? 
>> Was your CI environment in a clean state when running this test? Before the 
>> comment lots of things leaked.
> 
> 
> On my stretch laptop (i7-6820HQ CPU @ 2.70GHz (8 cores) with SSD)  the reported 
> problem is reproducible (I rerun 'ci_build.sh -q -f' several times in clean 
> state) it hung and with the less mount points (the mount points before and after 
> running are attached).
> 
> The strange thing that I observe two bitbake processes:
> 
> myo      26373  0.0  0.3 153116 29732 pts/0    Sl+  12:31   0:01 python3 
> /home/myo/work/isar/src/trunk/isar/bitbake/bin/bitbake 
> multiconfig:qemuarm-stretch:isar-image-base 
> multiconfig:qemuarm64-stretch:isar-image-base 
> multiconfig:qemuamd64-stretch:isar-image-base
> 
> myo      26379  2.5  0.6 328476 50028 ?        Sl   12:31   0:40 python3 
> /home/myo/work/isar/src/trunk/isar/bitbake/bin/bitbake 
> multiconfig:qemuarm-stretch:isar-image-base 
> multiconfig:qemuarm64-stretch:isar-image-base 
> multiconfig:qemuamd64-stretch:isar-image-base
> 

We run multiple bitbake sessions after each other. Maybe the first one never 
terminates (get stuck), and that is also why the rm after the first session 
fails. You need to stop the build there and analyse what is keeping the mount 
points busy.

> 
> When I turned on verbosity (called ci_build.sh without '-q' switch)
> 'scripts/ci_build -f' succeeded.
> 
> Build also fails on isar-build.org
> http://isar-build.org:8080/job/isar_mosipov_next/98/console:
> it complains  on
> 'rm: cannot remove 
> '/workspace/build/isar_mosipov_next/98/build/tmp/work/debian-stretch-armhf/isar-bootstrap-target/rootfs/dev': 
> Device or resource busy'
> 
> Note that ci-build.sh on isar-build.org is also called with '-q' switch.
> 
> 
> Questions:
> 
> 1) Did you launch ci_build with '-q' option?
> 

yes

> 2) Did you inspect mount points before and after launching ci_build? Are they 
> identical?
> 

yes

> 3) Do you run this in docker?
> 

yes, in all setups.

Jan

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

Re: [PATCH] isar-bootstrap: Fix and cleanup bind mounting

[Jan Kiszka]

On 04.12.18 15:24, [ext] Jan Kiszka wrote:
> On 04.12.18 11:49, Maxim Yu. Osipov wrote:
>> On 12/3/18 3:59 PM, Jan Kiszka wrote:
>>> On 30.11.18 10:20, Maxim Yu. Osipov wrote:
>>>> Hi Jan,
>>>>
>>>> I've just tried this patch (on the 'next' with reverted patch d40a9ac0) and 
>>>> ran "fast" CI
>>>>
>>>> isar$mount | wc -l
>>>> 34
>>>>
>>>> isar$./scripts/ci_build.sh -q -f
>>>>
>>>> CI script hung on CI stage when dpkg-base is modified
>>>> causing rebuilding recipes based on dpkg-base.
>>>>
>>>> The mount reports less (!) mount points than before launching the script.
>>>>
>>>> mount | wc -l
>>>> 31
>>>
>>> Any news on what's different on your side? Where exactly does your build 
>>> hang? Was your CI environment in a clean state when running this test? Before 
>>> the comment lots of things leaked.
>>
>>
>> On my stretch laptop (i7-6820HQ CPU @ 2.70GHz (8 cores) with SSD)  the 
>> reported problem is reproducible (I rerun 'ci_build.sh -q -f' several times in 
>> clean state) it hung and with the less mount points (the mount points before 
>> and after running are attached).
>>
>> The strange thing that I observe two bitbake processes:
>>
>> myo      26373  0.0  0.3 153116 29732 pts/0    Sl+  12:31   0:01 python3 
>> /home/myo/work/isar/src/trunk/isar/bitbake/bin/bitbake 
>> multiconfig:qemuarm-stretch:isar-image-base 
>> multiconfig:qemuarm64-stretch:isar-image-base 
>> multiconfig:qemuamd64-stretch:isar-image-base
>>
>> myo      26379  2.5  0.6 328476 50028 ?        Sl   12:31   0:40 python3 
>> /home/myo/work/isar/src/trunk/isar/bitbake/bin/bitbake 
>> multiconfig:qemuarm-stretch:isar-image-base 
>> multiconfig:qemuarm64-stretch:isar-image-base 
>> multiconfig:qemuamd64-stretch:isar-image-base
>>
> 
> We run multiple bitbake sessions after each other. Maybe the first one never 
> terminates (get stuck), and that is also why the rm after the first session 
> fails. You need to stop the build there and analyse what is keeping the mount 
> points busy.

Wait... If I terminate a build from inside the container (i.e. "natively") and 
then quickly try to delete the build artifacts, I can trigger that infamous 
empty /dev bug - on the host. That has always been the problem, and that is one 
reason why we encapsulate things into containers.

The reason for this is that bitbake's cooker waits for the last sub-process to 
finish before it calls the cleanup hook that does all the unmounting. If you 
delete something before that, you step into the mount point and purge its 
content. That /may/ be the issue here as well as we run rm directly after bitbake.

IOW: Possibly just a known limitation of current Isar design /wrt to unmounting 
in isar_handler() that now surfaces in CI. I would not be surprised you can 
resolve that by waiting for the last cooker instance to terminate before 
deleting tmp.

Jan

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

RE: [PATCH] isar-bootstrap: Fix and cleanup bind mounting

[Hombourger, Cedric]

Good catch & analysis Jan!
In our CI, our build script is checking for any mounts relative to the current directory before purging them

-----Original Message-----
From: Jan Kiszka [mailto:jan.kiszka@siemens.com] 
Sent: Tuesday, December 4, 2018 6:43 PM
To: Maxim Yu. Osipov <mosipov@ilbers.de>
Cc: Henning Schild <henning.schild@siemens.com>; isar-users <isar-users@googlegroups.com>; Hombourger, Cedric <Cedric_Hombourger@mentor.com>
Subject: Re: [PATCH] isar-bootstrap: Fix and cleanup bind mounting

On 04.12.18 15:24, [ext] Jan Kiszka wrote:
> On 04.12.18 11:49, Maxim Yu. Osipov wrote:
>> On 12/3/18 3:59 PM, Jan Kiszka wrote:
>>> On 30.11.18 10:20, Maxim Yu. Osipov wrote:
>>>> Hi Jan,
>>>>
>>>> I've just tried this patch (on the 'next' with reverted patch 
>>>> d40a9ac0) and ran "fast" CI
>>>>
>>>> isar$mount | wc -l
>>>> 34
>>>>
>>>> isar$./scripts/ci_build.sh -q -f
>>>>
>>>> CI script hung on CI stage when dpkg-base is modified causing 
>>>> rebuilding recipes based on dpkg-base.
>>>>
>>>> The mount reports less (!) mount points than before launching the script.
>>>>
>>>> mount | wc -l
>>>> 31
>>>
>>> Any news on what's different on your side? Where exactly does your 
>>> build hang? Was your CI environment in a clean state when running 
>>> this test? Before the comment lots of things leaked.
>>
>>
>> On my stretch laptop (i7-6820HQ CPU @ 2.70GHz (8 cores) with SSD)  
>> the reported problem is reproducible (I rerun 'ci_build.sh -q -f' 
>> several times in clean state) it hung and with the less mount points 
>> (the mount points before and after running are attached).
>>
>> The strange thing that I observe two bitbake processes:
>>
>> myo      26373  0.0  0.3 153116 29732 pts/0    Sl+  12:31   0:01 
>> python3 /home/myo/work/isar/src/trunk/isar/bitbake/bin/bitbake
>> multiconfig:qemuarm-stretch:isar-image-base
>> multiconfig:qemuarm64-stretch:isar-image-base
>> multiconfig:qemuamd64-stretch:isar-image-base
>>
>> myo      26379  2.5  0.6 328476 50028 ?        Sl   12:31   0:40 
>> python3 /home/myo/work/isar/src/trunk/isar/bitbake/bin/bitbake
>> multiconfig:qemuarm-stretch:isar-image-base
>> multiconfig:qemuarm64-stretch:isar-image-base
>> multiconfig:qemuamd64-stretch:isar-image-base
>>
> 
> We run multiple bitbake sessions after each other. Maybe the first one never 
> terminates (get stuck), and that is also why the rm after the first session 
> fails. You need to stop the build there and analyse what is keeping the mount 
> points busy.

Wait... If I terminate a build from inside the container (i.e. "natively") and 
then quickly try to delete the build artifacts, I can trigger that infamous 
empty /dev bug - on the host. That has always been the problem, and that is one 
reason why we encapsulate things into containers.

The reason for this is that bitbake's cooker waits for the last sub-process to 
finish before it calls the cleanup hook that does all the unmounting. If you 
delete something before that, you step into the mount point and purge its 
content. That /may/ be the issue here as well as we run rm directly after bitbake.

IOW: Possibly just a known limitation of current Isar design /wrt to unmounting 
in isar_handler() that now surfaces in CI. I would not be surprised you can 
resolve that by waiting for the last cooker instance to terminate before 
deleting tmp.

Jan

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

Re: [PATCH] isar-bootstrap: Fix and cleanup bind mounting

[Maxim Yu. Osipov]

Hi Jan, Cedric,

Another question:

Which debian/kernel do you use inside your VM/docker?

Is it also stretch?

The problem is reproducible at the the same point on stretch systems 
(with kernel SMP Debian 4.9.110-3+deb9u6 (2018-10-08) x86_64 GNU/Linux)

when commands are launched by hand:

Command 1)
bitbake  -c cache_base_repo multiconfig:qemuarm-stretch:isar-image-base 
multiconfig:qemuarm64-stretch:isar-image-base multiconfig:qemuamd64-
stretch:isar-image-base

Command 2)
sudo rm -rf tmp

Command 3)
sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO 
?= "1"/g' conf/local.conf

No problems detected at this point - the same mounts etc.

The next command hungs on the last task (according strace bitbake tries 
to unmount /sys /dev)

Command 4)
bitbake multiconfig:qemuarm-stretch:isar-image-base 
multiconfig:qemuarm64-stretch:isar-image-base 
multiconfig:qemuamd64-stretch:isar-image-base

Maxim.

On 12/4/18 6:45 PM, Hombourger, Cedric wrote:
> Good catch & analysis Jan!
> In our CI, our build script is checking for any mounts relative to the current directory before purging them
> 
> -----Original Message-----
> From: Jan Kiszka [mailto:jan.kiszka@siemens.com]
> Sent: Tuesday, December 4, 2018 6:43 PM
> To: Maxim Yu. Osipov <mosipov@ilbers.de>
> Cc: Henning Schild <henning.schild@siemens.com>; isar-users <isar-users@googlegroups.com>; Hombourger, Cedric <Cedric_Hombourger@mentor.com>
> Subject: Re: [PATCH] isar-bootstrap: Fix and cleanup bind mounting
> 
> On 04.12.18 15:24, [ext] Jan Kiszka wrote:
>> On 04.12.18 11:49, Maxim Yu. Osipov wrote:
>>> On 12/3/18 3:59 PM, Jan Kiszka wrote:
>>>> On 30.11.18 10:20, Maxim Yu. Osipov wrote:
>>>>> Hi Jan,
>>>>>
>>>>> I've just tried this patch (on the 'next' with reverted patch
>>>>> d40a9ac0) and ran "fast" CI
>>>>>
>>>>> isar$mount | wc -l
>>>>> 34
>>>>>
>>>>> isar$./scripts/ci_build.sh -q -f
>>>>>
>>>>> CI script hung on CI stage when dpkg-base is modified causing
>>>>> rebuilding recipes based on dpkg-base.
>>>>>
>>>>> The mount reports less (!) mount points than before launching the script.
>>>>>
>>>>> mount | wc -l
>>>>> 31
>>>>
>>>> Any news on what's different on your side? Where exactly does your
>>>> build hang? Was your CI environment in a clean state when running
>>>> this test? Before the comment lots of things leaked.
>>>
>>>
>>> On my stretch laptop (i7-6820HQ CPU @ 2.70GHz (8 cores) with SSD)
>>> the reported problem is reproducible (I rerun 'ci_build.sh -q -f'
>>> several times in clean state) it hung and with the less mount points
>>> (the mount points before and after running are attached).
>>>
>>> The strange thing that I observe two bitbake processes:
>>>
>>> myo      26373  0.0  0.3 153116 29732 pts/0    Sl+  12:31   0:01
>>> python3 /home/myo/work/isar/src/trunk/isar/bitbake/bin/bitbake
>>> multiconfig:qemuarm-stretch:isar-image-base
>>> multiconfig:qemuarm64-stretch:isar-image-base
>>> multiconfig:qemuamd64-stretch:isar-image-base
>>>
>>> myo      26379  2.5  0.6 328476 50028 ?        Sl   12:31   0:40
>>> python3 /home/myo/work/isar/src/trunk/isar/bitbake/bin/bitbake
>>> multiconfig:qemuarm-stretch:isar-image-base
>>> multiconfig:qemuarm64-stretch:isar-image-base
>>> multiconfig:qemuamd64-stretch:isar-image-base
>>>
>>
>> We run multiple bitbake sessions after each other. Maybe the first one never
>> terminates (get stuck), and that is also why the rm after the first session
>> fails. You need to stop the build there and analyse what is keeping the mount
>> points busy.
> 
> Wait... If I terminate a build from inside the container (i.e. "natively") and
> then quickly try to delete the build artifacts, I can trigger that infamous
> empty /dev bug - on the host. That has always been the problem, and that is one
> reason why we encapsulate things into containers.
> 
> The reason for this is that bitbake's cooker waits for the last sub-process to
> finish before it calls the cleanup hook that does all the unmounting. If you
> delete something before that, you step into the mount point and purge its
> content. That /may/ be the issue here as well as we run rm directly after bitbake.
> 
> IOW: Possibly just a known limitation of current Isar design /wrt to unmounting
> in isar_handler() that now surfaces in CI. I would not be surprised you can
> resolve that by waiting for the last cooker instance to terminate before
> deleting tmp.
> 
> Jan
> 


-- 
Maxim Osipov
ilbers GmbH
Maria-Merian-Str. 8
85521 Ottobrunn
Germany
+49 (151) 6517 6917
mosipov@ilbers.de
http://ilbers.de/
Commercial register Munich, HRB 214197
General Manager: Baurzhan Ismagulov

Re: [PATCH] isar-bootstrap: Fix and cleanup bind mounting

[Jan Kiszka]

On 04.12.18 17:59, Maxim Yu. Osipov wrote:
> Hi Jan, Cedric,
> 
> Another question:
> 
> Which debian/kernel do you use inside your VM/docker?

Container is kasproject/kas-isar:latest here, kernel is $random-host.

> 
> Is it also stretch?

Yes.

> 
> The problem is reproducible at the the same point on stretch systems (with 
> kernel SMP Debian 4.9.110-3+deb9u6 (2018-10-08) x86_64 GNU/Linux)
> 
> when commands are launched by hand:

What was your Isar baseline for that? I tested with this branch:

https://github.com/siemens/isar/commit/03e394b2aa58b5b1404fde76881774baf4a541bc

> 
> Command 1)
> bitbake  -c cache_base_repo multiconfig:qemuarm-stretch:isar-image-base 
> multiconfig:qemuarm64-stretch:isar-image-base multiconfig:qemuamd64-
> stretch:isar-image-base

Did you check if everything was unmounted at this point already?

> 
> Command 2)
> sudo rm -rf tmp
> 
> Command 3)
> sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO ?= 
> "1"/g' conf/local.conf
> 
> No problems detected at this point - the same mounts etc.
> 
> The next command hungs on the last task (according strace bitbake tries to 
> unmount /sys /dev)

And who is holding back that mounts (lsof)? How does pstree -apl look like?

BTW, can you recover the build system for this stage? As I said, one of the 
symptoms or side-effect is removal of device nodes on those side when /dev is 
mounted at the wrong time.

Jan

> 
> Command 4)
> bitbake multiconfig:qemuarm-stretch:isar-image-base 
> multiconfig:qemuarm64-stretch:isar-image-base 
> multiconfig:qemuamd64-stretch:isar-image-base
> 
> Maxim.
> 
> On 12/4/18 6:45 PM, Hombourger, Cedric wrote:
>> Good catch & analysis Jan!
>> In our CI, our build script is checking for any mounts relative to the current 
>> directory before purging them
>>
>> -----Original Message-----
>> From: Jan Kiszka [mailto:jan.kiszka@siemens.com]
>> Sent: Tuesday, December 4, 2018 6:43 PM
>> To: Maxim Yu. Osipov <mosipov@ilbers.de>
>> Cc: Henning Schild <henning.schild@siemens.com>; isar-users 
>> <isar-users@googlegroups.com>; Hombourger, Cedric <Cedric_Hombourger@mentor.com>
>> Subject: Re: [PATCH] isar-bootstrap: Fix and cleanup bind mounting
>>
>> On 04.12.18 15:24, [ext] Jan Kiszka wrote:
>>> On 04.12.18 11:49, Maxim Yu. Osipov wrote:
>>>> On 12/3/18 3:59 PM, Jan Kiszka wrote:
>>>>> On 30.11.18 10:20, Maxim Yu. Osipov wrote:
>>>>>> Hi Jan,
>>>>>>
>>>>>> I've just tried this patch (on the 'next' with reverted patch
>>>>>> d40a9ac0) and ran "fast" CI
>>>>>>
>>>>>> isar$mount | wc -l
>>>>>> 34
>>>>>>
>>>>>> isar$./scripts/ci_build.sh -q -f
>>>>>>
>>>>>> CI script hung on CI stage when dpkg-base is modified causing
>>>>>> rebuilding recipes based on dpkg-base.
>>>>>>
>>>>>> The mount reports less (!) mount points than before launching the script.
>>>>>>
>>>>>> mount | wc -l
>>>>>> 31
>>>>>
>>>>> Any news on what's different on your side? Where exactly does your
>>>>> build hang? Was your CI environment in a clean state when running
>>>>> this test? Before the comment lots of things leaked.
>>>>
>>>>
>>>> On my stretch laptop (i7-6820HQ CPU @ 2.70GHz (8 cores) with SSD)
>>>> the reported problem is reproducible (I rerun 'ci_build.sh -q -f'
>>>> several times in clean state) it hung and with the less mount points
>>>> (the mount points before and after running are attached).
>>>>
>>>> The strange thing that I observe two bitbake processes:
>>>>
>>>> myo      26373  0.0  0.3 153116 29732 pts/0    Sl+  12:31   0:01
>>>> python3 /home/myo/work/isar/src/trunk/isar/bitbake/bin/bitbake
>>>> multiconfig:qemuarm-stretch:isar-image-base
>>>> multiconfig:qemuarm64-stretch:isar-image-base
>>>> multiconfig:qemuamd64-stretch:isar-image-base
>>>>
>>>> myo      26379  2.5  0.6 328476 50028 ?        Sl   12:31   0:40
>>>> python3 /home/myo/work/isar/src/trunk/isar/bitbake/bin/bitbake
>>>> multiconfig:qemuarm-stretch:isar-image-base
>>>> multiconfig:qemuarm64-stretch:isar-image-base
>>>> multiconfig:qemuamd64-stretch:isar-image-base
>>>>
>>>
>>> We run multiple bitbake sessions after each other. Maybe the first one never
>>> terminates (get stuck), and that is also why the rm after the first session
>>> fails. You need to stop the build there and analyse what is keeping the mount
>>> points busy.
>>
>> Wait... If I terminate a build from inside the container (i.e. "natively") and
>> then quickly try to delete the build artifacts, I can trigger that infamous
>> empty /dev bug - on the host. That has always been the problem, and that is one
>> reason why we encapsulate things into containers.
>>
>> The reason for this is that bitbake's cooker waits for the last sub-process to
>> finish before it calls the cleanup hook that does all the unmounting. If you
>> delete something before that, you step into the mount point and purge its
>> content. That /may/ be the issue here as well as we run rm directly after 
>> bitbake.
>>
>> IOW: Possibly just a known limitation of current Isar design /wrt to unmounting
>> in isar_handler() that now surfaces in CI. I would not be surprised you can
>> resolve that by waiting for the last cooker instance to terminate before
>> deleting tmp.
>>
>> Jan
>>
> 
> 

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

Re: [PATCH] isar-bootstrap: Fix and cleanup bind mounting

[Jan Kiszka]

On 04.12.18 18:10, Jan Kiszka wrote:
> On 04.12.18 17:59, Maxim Yu. Osipov wrote:
>> Hi Jan, Cedric,
>>
>> Another question:
>>
>> Which debian/kernel do you use inside your VM/docker?
> 
> Container is kasproject/kas-isar:latest here, kernel is $random-host.
> 
>>
>> Is it also stretch?
> 
> Yes.
> 
>>
>> The problem is reproducible at the the same point on stretch systems (with 
>> kernel SMP Debian 4.9.110-3+deb9u6 (2018-10-08) x86_64 GNU/Linux)
>>
>> when commands are launched by hand:
> 
> What was your Isar baseline for that? I tested with this branch:
> 
> https://github.com/siemens/isar/commit/03e394b2aa58b5b1404fde76881774baf4a541bc
> 
>>
>> Command 1)
>> bitbake  -c cache_base_repo multiconfig:qemuarm-stretch:isar-image-base 
>> multiconfig:qemuarm64-stretch:isar-image-base multiconfig:qemuamd64-
>> stretch:isar-image-base
> 
> Did you check if everything was unmounted at this point already?
> 
>>
>> Command 2)
>> sudo rm -rf tmp
>>
>> Command 3)
>> sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO ?= 
>> "1"/g' conf/local.conf
>>
>> No problems detected at this point - the same mounts etc.
>>
>> The next command hungs on the last task (according strace bitbake tries to 
>> unmount /sys /dev)
> 
> And who is holding back that mounts (lsof)? How does pstree -apl look like?
> 
> BTW, can you recover the build system for this stage? As I said, one of the 
> symptoms or side-effect is removal of device nodes on those side when /dev is 
> mounted at the wrong time.
> 

It's also worth trying if any of these hunks help:

diff --git a/meta/classes/isar-events.bbclass b/meta/classes/isar-events.bbclass
index b3ce434..b2607b6 100644
--- a/meta/classes/isar-events.bbclass
+++ b/meta/classes/isar-events.bbclass
@@ -18,7 +18,7 @@ python isar_handler () {
         with open('/proc/mounts', 'rU') as f:
             for line in f:
                 if basepath in line:
-                    if subprocess.call('sudo umount ' + line.split()[1],
+                    if subprocess.call('sudo umount -l ' + line.split()[1],
                                        stdout=devnull, stderr=devnull,
                                        shell=True) != 0:
                         result = False
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index 20b77b7..aae9084 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -92,6 +92,7 @@ if [ -n "$FAST_BUILD" ]; then
         multiconfig:qemuarm-stretch:isar-image-base \
         multiconfig:qemuarm64-stretch:isar-image-base \
         multiconfig:qemuamd64-stretch:isar-image-base
+    while [ -e bitbake.sock ]; do sleep 1; done
     sudo rm -rf tmp
     sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
     bitbake $BB_ARGS \


Jan

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

Re: [PATCH] isar-bootstrap: Fix and cleanup bind mounting

[Jan Kiszka]

On 04.12.18 16:42, Jan Kiszka wrote:
> On 04.12.18 15:24, [ext] Jan Kiszka wrote:
>> On 04.12.18 11:49, Maxim Yu. Osipov wrote:
>>> On 12/3/18 3:59 PM, Jan Kiszka wrote:
>>>> On 30.11.18 10:20, Maxim Yu. Osipov wrote:
>>>>> Hi Jan,
>>>>>
>>>>> I've just tried this patch (on the 'next' with reverted patch d40a9ac0) and 
>>>>> ran "fast" CI
>>>>>
>>>>> isar$mount | wc -l
>>>>> 34
>>>>>
>>>>> isar$./scripts/ci_build.sh -q -f
>>>>>
>>>>> CI script hung on CI stage when dpkg-base is modified
>>>>> causing rebuilding recipes based on dpkg-base.
>>>>>
>>>>> The mount reports less (!) mount points than before launching the script.
>>>>>
>>>>> mount | wc -l
>>>>> 31
>>>>
>>>> Any news on what's different on your side? Where exactly does your build 
>>>> hang? Was your CI environment in a clean state when running this test? 
>>>> Before the comment lots of things leaked.
>>>
>>>
>>> On my stretch laptop (i7-6820HQ CPU @ 2.70GHz (8 cores) with SSD)  the 
>>> reported problem is reproducible (I rerun 'ci_build.sh -q -f' several times 
>>> in clean state) it hung and with the less mount points (the mount points 
>>> before and after running are attached).
>>>
>>> The strange thing that I observe two bitbake processes:
>>>
>>> myo      26373  0.0  0.3 153116 29732 pts/0    Sl+  12:31   0:01 python3 
>>> /home/myo/work/isar/src/trunk/isar/bitbake/bin/bitbake 
>>> multiconfig:qemuarm-stretch:isar-image-base 
>>> multiconfig:qemuarm64-stretch:isar-image-base 
>>> multiconfig:qemuamd64-stretch:isar-image-base
>>>
>>> myo      26379  2.5  0.6 328476 50028 ?        Sl   12:31   0:40 python3 
>>> /home/myo/work/isar/src/trunk/isar/bitbake/bin/bitbake 
>>> multiconfig:qemuarm-stretch:isar-image-base 
>>> multiconfig:qemuarm64-stretch:isar-image-base 
>>> multiconfig:qemuamd64-stretch:isar-image-base
>>>
>>
>> We run multiple bitbake sessions after each other. Maybe the first one never 
>> terminates (get stuck), and that is also why the rm after the first session 
>> fails. You need to stop the build there and analyse what is keeping the mount 
>> points busy.
> 
> Wait... If I terminate a build from inside the container (i.e. "natively") and 
> then quickly try to delete the build artifacts, I can trigger that infamous 
> empty /dev bug - on the host. That has always been the problem, and that is one 
> reason why we encapsulate things into containers.
> 
> The reason for this is that bitbake's cooker waits for the last sub-process to 
> finish before it calls the cleanup hook that does all the unmounting. If you 
> delete something before that, you step into the mount point and purge its 
> content. That /may/ be the issue here as well as we run rm directly after bitbake.
> 
> IOW: Possibly just a known limitation of current Isar design /wrt to unmounting 
> in isar_handler() that now surfaces in CI. I would not be surprised you can 
> resolve that by waiting for the last cooker instance to terminate before 
> deleting tmp.
>

...or we are just missing "-l" as umount parameter isar_handler(), like Claudius 
added elsewhere. Though that may not conceptually resolve the race window 
between bitbake terminating and cooker still running the handler.

Jan

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

Re: [PATCH] isar-bootstrap: Fix and cleanup bind mounting

[Maxim Yu. Osipov]

On 11/29/18 7:29 PM, Jan Kiszka wrote:
> First, wrong variables were used (there is no BUILDCHROOT_DIR for
> isar_bootstrap). And then we simply left the mount points active after
> completing the bootstrap. The led to setup_root_file_system copying all
> the mount point content over into the buildchroot or rootfs - including
> base-apt when it was used.

Applied to the 'next',

Thanks,
Maxim.

> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
> ---
> 
> This is a precondition for "isar-image: umount base-apt when doing
> offline build" to work properly. IOW, that patch reveal these bugs.
> 
> Possibly, this also explains the increasing mount count that was visible
> in CI.
> 
>   meta/recipes-core/isar-bootstrap/isar-bootstrap.inc | 21 +++++++++++----------
>   1 file changed, 11 insertions(+), 10 deletions(-)
> 
> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> index 8ea3b08..da077d0 100644
> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> @@ -185,12 +185,8 @@ isar_bootstrap() {
>       sudo -E flock "${ISAR_BOOTSTRAP_LOCK}" -c "\
>           set -e
>           if [ ! -e "${DEPLOY_ISAR_BOOTSTRAP}" ]; then
> -            if [ -e "${ROOTFSDIR}" ]; then
> -               umount -R -l "${ROOTFSDIR}/dev" || true
> -               umount -l "${ROOTFSDIR}/proc" || true
> -               umount -R -l "${ROOTFSDIR}/sys" || true
> -               rm -rf "${ROOTFSDIR}"
> -            fi
> +            rm -rf "${ROOTFSDIR}"
> +
>               if [ ${IS_HOST} ]; then
>                   ${DEBOOTSTRAP} $debootstrap_args \
>                                  ${@get_distro_components_argument(d, True)} \
> @@ -247,17 +243,22 @@ isar_bootstrap() {
>               "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
>   
>               # update APT
> -            mount --rbind /dev ${BUILDCHROOT_DIR}/dev
> -            mount --make-rslave ${BUILDCHROOT_DIR}/dev
> +            mount --rbind /dev ${ROOTFSDIR}/dev
> +            mount --make-rslave ${ROOTFSDIR}/dev
>               mount -t proc none ${ROOTFSDIR}/proc
> -            mount --rbind /sys ${BUILDCHROOT_DIR}/sys
> -            mount --make-rslave ${BUILDCHROOT_DIR}/sys
> +            mount --rbind /sys ${ROOTFSDIR}/sys
> +            mount --make-rslave ${ROOTFSDIR}/sys
>   
>               export DEBIAN_FRONTEND=noninteractive
>               chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
>               chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
>                                     -o Debug::pkgProblemResolver=yes
>   
> +            umount -R -l "${ROOTFSDIR}/dev"
> +            umount -l "${ROOTFSDIR}/proc"
> +            umount -R -l "${ROOTFSDIR}/sys"
> +            umount -l "${ROOTFSDIR}/base-apt" || true
> +
>               # Finalize debootstrap by setting the link in deploy
>               ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
>           fi"
> 


-- 
Maxim Osipov
ilbers GmbH
Maria-Merian-Str. 8
85521 Ottobrunn
Germany
+49 (151) 6517 6917
mosipov@ilbers.de
http://ilbers.de/
Commercial register Munich, HRB 214197
General Manager: Baurzhan Ismagulov