[PATCH v3] deb-dl-dir: Fallback to copying when import-export

[PATCH v3] deb-dl-dir: Fallback to copying when import-export

[Uladzimir Bely]

By default, hardlinks are used when doing DL_DIR import and export.
This fails if DL_DIR's filesystem differs from TMPDIR's one.

Fallback to file copying (with lower performance and huge disk usage)
in this case.

Also rework sudo using when import/export by preferring "sudo/while"
approach instead of "while/sudo" (less spam in system logs).

Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
---
 meta/classes/deb-dl-dir.bbclass | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

Changes since v2:
 - Reworked sudo usage approach from "while/sudo" to "sudo/while".

Changes since v1:
 - Try create handrlink first and then copy if failed. It should better
handle hardlink issues that may happen even on the same partition.

diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index 7db25251..ca2a1ee2 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -81,20 +81,24 @@ deb_dl_dir_import() {
     export rootfs="${1}"
     sudo mkdir -p "${rootfs}"/var/cache/apt/archives/
     [ ! -d "${pc}" ] && return 0
-    flock -s "${pc}".lock -c '
+    flock -s "${pc}".lock sudo -Es << 'EOSUDO'
         set -e
         printenv | grep -q BB_VERBOSE_LOGS && set -x
 
-        sudo find "${pc}" -type f -iname "*\.deb" -exec \
-            ln -Pf -t "${rootfs}"/var/cache/apt/archives/ {} +
-    '
+        find "${pc}" -type f -iname "*\.deb" |\
+        while read p; do
+            ln -Pf -t "${rootfs}"/var/cache/apt/archives/ "$p" 2>/dev/null ||
+                cp -n --no-preserve=owner -t "${rootfs}"/var/cache/apt/archives/ "$p"
+        done
+EOSUDO
 }
 
 deb_dl_dir_export() {
     export pc="${DEBDIR}/${2}"
     export rootfs="${1}"
+    export owner=$(id -u):$(id -g)
     mkdir -p "${pc}"
-    flock "${pc}".lock -c '
+    flock "${pc}".lock sudo -Es << 'EOSUDO'
         set -e
         printenv | grep -q BB_VERBOSE_LOGS && set -x
 
@@ -109,8 +113,9 @@ deb_dl_dir_export() {
             if [ -n "$package" ]; then
                 cmp --silent "$package" "$p" && continue
             fi
-            sudo ln -Pf "${p}" "${pc}"
+            ln -Pf "${p}" "${pc}" 2>/dev/null ||
+                cp -n "${p}" "${pc}"
         done
-        sudo chown -R $(id -u):$(id -g) "${pc}"
-    '
+        chown -R ${owner} "${pc}"
+EOSUDO
 }
-- 
2.20.1

Re: [PATCH v3] deb-dl-dir: Fallback to copying when import-export

[Uladzimir Bely]

In the email from Monday, 13 February 2023 10:44:39 +03 user Uladzimir Bely 
wrote:
> By default, hardlinks are used when doing DL_DIR import and export.
> This fails if DL_DIR's filesystem differs from TMPDIR's one.
> 
> Fallback to file copying (with lower performance and huge disk usage)
> in this case.
> 
> Also rework sudo using when import/export by preferring "sudo/while"
> approach instead of "while/sudo" (less spam in system logs).
> 
> Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
> ---
>  meta/classes/deb-dl-dir.bbclass | 21 +++++++++++++--------
>  1 file changed, 13 insertions(+), 8 deletions(-)
> 
> Changes since v2:
>  - Reworked sudo usage approach from "while/sudo" to "sudo/while".
> 
> Changes since v1:
>  - Try create handrlink first and then copy if failed. It should better
> handle hardlink issues that may happen even on the same partition.
> 
> diff --git a/meta/classes/deb-dl-dir.bbclass
> b/meta/classes/deb-dl-dir.bbclass index 7db25251..ca2a1ee2 100644
> --- a/meta/classes/deb-dl-dir.bbclass
> +++ b/meta/classes/deb-dl-dir.bbclass
> @@ -81,20 +81,24 @@ deb_dl_dir_import() {
>      export rootfs="${1}"
>      sudo mkdir -p "${rootfs}"/var/cache/apt/archives/
>      [ ! -d "${pc}" ] && return 0
> -    flock -s "${pc}".lock -c '
> +    flock -s "${pc}".lock sudo -Es << 'EOSUDO'
>          set -e
>          printenv | grep -q BB_VERBOSE_LOGS && set -x
> 
> -        sudo find "${pc}" -type f -iname "*\.deb" -exec \
> -            ln -Pf -t "${rootfs}"/var/cache/apt/archives/ {} +
> -    '
> +        find "${pc}" -type f -iname "*\.deb" |\
> +        while read p; do
> +            ln -Pf -t "${rootfs}"/var/cache/apt/archives/ "$p" 2>/dev/null
> || +                cp -n --no-preserve=owner -t
> "${rootfs}"/var/cache/apt/archives/ "$p" +        done
> +EOSUDO
>  }
> 
>  deb_dl_dir_export() {
>      export pc="${DEBDIR}/${2}"
>      export rootfs="${1}"
> +    export owner=$(id -u):$(id -g)
>      mkdir -p "${pc}"
> -    flock "${pc}".lock -c '
> +    flock "${pc}".lock sudo -Es << 'EOSUDO'
>          set -e
>          printenv | grep -q BB_VERBOSE_LOGS && set -x
> 
> @@ -109,8 +113,9 @@ deb_dl_dir_export() {
>              if [ -n "$package" ]; then
>                  cmp --silent "$package" "$p" && continue
>              fi
> -            sudo ln -Pf "${p}" "${pc}"
> +            ln -Pf "${p}" "${pc}" 2>/dev/null ||
> +                cp -n "${p}" "${pc}"
>          done
> -        sudo chown -R $(id -u):$(id -g) "${pc}"
> -    '
> +        chown -R ${owner} "${pc}"
> +EOSUDO
>  }

Applied to next