[PATCH v7] image tools ext.: start_imager_session not break the rebuild, v7

[PATCH v7] image tools ext.: start_imager_session not break the rebuild, v7

[roberto.foglietta]

From: "Roberto A. Foglietta" <roberto.foglietta@gmail.com>

image tools extension, start_imager_session, bugfix and improvment

The original Ilbers ISAR code was buggy but the 1st fix do not improve
the performances nore solved completely the problem which has been
defintely addressed in part.2 - then this p.3 aims to improve the
performance brutally deleting the closing schroot session without relies
on the slower and sometimes failing schroot -e python script.

p.1: bugfix, initial
p.2: bugfix completed, code rationalisation
p.3: performance improvement bypassing schroot -e and Co.
v.4: the squash of the three parts
v.5: deep reworking
v.6: bugfixing
v.7: removed the code left behind but not used

Signed-off-by: Roberto A. Foglietta <roberto.foglietta@gmail.com>
---
 meta/classes/image-tools-extension.bbclass | 39 ++++++++--------------
 meta/classes/isar-events.bbclass           | 31 +++++++++++------
 meta/classes/sbuild.bbclass                |  3 +-
 3 files changed, 37 insertions(+), 36 deletions(-)

diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass
index 48a93c8..9c46b65 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass
@@ -60,18 +60,20 @@ python do_start_imager_session() {
     sbuild_chroot = d.getVar("SBUILD_CHROOT", True)
     session_id = d.getVar("IMAGER_SCHROOT_SESSION_ID", True)
     try:
-        if subprocess.run("schroot -l --all-sessions | grep %s" % session_id, shell=True).returncode:
-            subprocess.run("schroot -b -c %s -n %s" % (sbuild_chroot, session_id), shell=True, check=True)
-            bb.debug(2, "Open schroot session %s" % session_id)
-        else:
-            subprocess.run("schroot --recover-session -c %s" % session_id, shell=True, check=True)
-            bb.debug(2, "Reuse schroot session %s" % session_id)
-        d.setVar("SCHROOT_OPEN_SESSION_ID", session_id)
+        bb.debug(2, "Opening schroot session %s" % sbuild_chroot)
+        id = subprocess.run("schroot -d / -b -c %s -n %s -- printenv -0 SCHROOT_ALIAS_NAME"
+            % (sbuild_chroot, session_id), shell=True, check=True)
     except subprocess.CalledProcessError as err:
-        subprocess.run("schroot -e -c %s" % session_id, shell=True)
-        bb.build.exec_func("remove_mounts", d)
-        bb.build.exec_func("schroot_delete_configs", d)
-        bb.fatal("Could not create schroot session: %s" % err.output.decode('utf-8') if err.output else "")
+        try:
+            bb.debug(2, "Reusing schroot session %s" % sbuild_chroot)
+            id = subprocess.run("schroot -d / -r -c %s -- printenv -0 SCHROOT_ALIAS_NAME"
+                % session_id, shell=True, check=True)
+        except subprocess.CalledProcessError as err:
+            bb.debug(2, "Closing schroot session %s (%s)" % (sbuild_chroot, session_id))
+#           bb.build.exec_func("stop_imager_session", d)
+            bb.build.exec_func("stop_schroot_session", d)
+    if 'id' in locals():
+        d.setVar("SBUILD_CHROOT", id)
 }
 addtask start_imager_session before do_stop_imager_session after do_rootfs_finalize
 
@@ -79,20 +81,7 @@ do_stop_imager_session[depends] = "${SCHROOT_DEP}"
 do_stop_imager_session[nostamp] = "1"
 do_stop_imager_session[network] = "${TASK_USE_SUDO}"
 python do_stop_imager_session() {
-    import subprocess
-    session_id = d.getVar("IMAGER_SCHROOT_SESSION_ID", True)
-    try:
-        id = subprocess.run("schroot -d / -r -c %s -- printenv -0 SCHROOT_ALIAS_NAME" % session_id,
-                            shell=True, check=True, stdout=subprocess.PIPE).stdout.decode('utf-8')
-        bb.debug(2, "Close schroot session %s (%s)" % (session_id, id))
-        subprocess.run("schroot -e -c %s" % session_id, shell=True, check=True)
-    except subprocess.CalledProcessError as err:
-        bb.error("Could not close schroot session %s: %s" % (session_id, err.output.decode('utf-8')) if err.output else "")
-    finally:
-        if 'id' in locals():
-            d.setVar("SBUILD_CHROOT", id)
-            bb.build.exec_func("remove_mounts", d)
-            bb.build.exec_func("schroot_delete_configs", d)
+    bb.build.exec_func("stop_schroot_session", d)
 }
 addtask stop_imager_session before do_deploy after do_image
 
diff --git a/meta/classes/isar-events.bbclass b/meta/classes/isar-events.bbclass
index 5343d4b..7f68ffa 100644
--- a/meta/classes/isar-events.bbclass
+++ b/meta/classes/isar-events.bbclass
@@ -42,6 +42,25 @@ task_failed[eventmask] = "bb.build.TaskFailed"
 
 addhandler build_completed
 
+python stop_schroot_session() {
+    session_id = d.getVar("IMAGER_SCHROOT_SESSION_ID", True)
+    import subprocess
+    try:
+        id = subprocess.run("schroot -d / -r -c %s -- printenv -0 SCHROOT_ALIAS_NAME" % session_id,
+                            shell=True, check=True, stdout=subprocess.PIPE).stdout.decode('utf-8')
+        bb.debug(2, "Close schroot session %s (%s)" % (session_id, id))
+        subprocess.run("schroot -e -c %s" % session_id, shell=True, check=True)
+        d.setVar("SBUILD_CHROOT", id)
+        bb.build.exec_func("remove_mounts", d)
+        bb.build.exec_func("schroot_delete_configs", d)
+    except subprocess.CalledProcessError as err:
+        bb.error(2, "Could not close schroot session %s: %s"
+            % (session_id, err.output.decode('utf-8')) if err.output else "")
+        udir = d.getVar("SCHROOT_OVERLAY_DIR", True)
+        subprocess.run("sudo rm -rf --one-file-system %s/%s /var/lib/schroot/session/%s"
+            % (udir, session_id, session_id), shell=True)
+}
+
 python build_completed() {
     import subprocess
 
@@ -56,16 +75,8 @@ python build_completed() {
                               shell=True, stdout=subprocess.PIPE).stdout.decode('utf-8')
     for line in sessions.splitlines():
         session_id = line.split(':', 1)[1]
-        bb.debug(1, 'Closing imager session %s' % session_id)
-        id = subprocess.run("schroot -d / -r -c %s -- printenv -0 SCHROOT_ALIAS_NAME" % session_id,
-                            shell=True, check=True, stdout=subprocess.PIPE).stdout.decode('utf-8')
-        if id:
-            subprocess.run('schroot --recover-session -c %s' % session_id, shell=True, check=True)
-            subprocess.run('schroot -e -c %s' % session_id, shell=True, check=True)
-        if 'id' in locals():
-            d.setVar('SBUILD_CHROOT', id)
-            bb.build.exec_func('remove_mounts', d)
-            bb.build.exec_func('schroot_delete_configs', d)
+        bb.debug(2, 'Closing imager session %s' % session_id)
+        bb.build.exec_func("stop_schroot_session", d)
 
     with open('/proc/mounts') as f:
         for line in f.readlines():
diff --git a/meta/classes/sbuild.bbclass b/meta/classes/sbuild.bbclass
index 06c01d6..25c3e08 100644
--- a/meta/classes/sbuild.bbclass
+++ b/meta/classes/sbuild.bbclass
@@ -26,6 +26,7 @@ SBUILD_CHROOT ?= "${DEBDISTRONAME}-${SCHROOT_USER}-${ISAR_BUILD_UUID}-${@os.getp
 
 SBUILD_CONF_DIR ?= "${SCHROOT_CONF}/${SBUILD_CHROOT}"
 SCHROOT_CONF_FILE ?= "${SCHROOT_CONF}/chroot.d/${SBUILD_CHROOT}"
+SCHROOT_OVERLAY_DIR ?= "${TMPDIR}/schroot-overlay"
 
 SBUILD_CONFIG="${WORKDIR}/sbuild.conf"
 
@@ -54,7 +55,7 @@ root-groups=root,sbuild
 source-root-users=${SCHROOT_USER}
 source-root-groups=root,sbuild
 union-type=overlay
-union-overlay-directory=${TMPDIR}/schroot-overlay
+union-overlay-directory=${SCHROOT_OVERLAY_DIR}
 preserve-environment=true
 EOF
 
-- 
2.34.1

Re: [PATCH v7] image tools ext.: start_imager_session not break the rebuild, v7

[Roberto A. Foglietta]

On Tue, 10 Jan 2023 at 05:24, <roberto.foglietta@linuxteam.org> wrote:
>
> From: "Roberto A. Foglietta" <roberto.foglietta@gmail.com>
>
> image tools extension, start_imager_session, bugfix and improvement
>
> The original Ilbers ISAR code was buggy but the 1st fix do not improve
> the performances nore solved completely the problem which has been
> definitely addressed in part.2 - then this p.3 aims to improve the
> performance brutally deleting the closing schroot session without relies
> on the slower and sometimes failing schroot -e python script.
>
> p.1: bugfix, initial
> p.2: bugfix completed, code rationalisation
> p.3: performance improvement bypassing schroot -e and Co.
> v.4: the squash of the three parts
> v.5: deep reworking
> v.6: bug fixing
> v.7: removed the code left behind but not used
>
> Signed-off-by: Roberto A. Foglietta <roberto.foglietta@gmail.com>
> ---
>  meta/classes/image-tools-extension.bbclass | 39 ++++++++--------------
>  meta/classes/isar-events.bbclass           | 31 +++++++++++------
>  meta/classes/sbuild.bbclass                |  3 +-
>  3 files changed, 37 insertions(+), 36 deletions(-)
>

This patch has no impact on building time (1 sec or less) nor disk
usage or storage.

Best regards, R-