[PATCH 1/2] container_fetcher: Fix missing checksum warning

[PATCH 1/2] container_fetcher: Fix missing checksum warning

['Clara Kowalsky' via isar-users]

In case only a tag is specified for a container image in the SRC_URI and
no digest, a warning should be issued with the recommendation to add the
digest of the container image.
So far, the number specified in the warning would be the checksum of the
manifest.json, which is a metadata file. However, we want to show the
registry digest, which is calculated over the complete image content.
In addition, reading the manifest.json does not work at this point
anyway, as skopeo has already packed it into a Docker archive.

Signed-off-by: Clara Kowalsky <clara.kowalsky@siemens.com>
---
 meta/lib/container_fetcher.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/meta/lib/container_fetcher.py b/meta/lib/container_fetcher.py
index 0d659154..16467abb 100644
--- a/meta/lib/container_fetcher.py
+++ b/meta/lib/container_fetcher.py
@@ -6,6 +6,7 @@
 import oe.path
 import os
 import tempfile
+import json
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import logger
 from   bb.fetch2 import MissingChecksumEvent
@@ -60,16 +61,17 @@ class Container(FetchMethod):
         if ud.digest:
             return
 
-        checksum = bb.utils.sha256_file(ud.localpath + "/manifest.json")
-        checksum_line = f"SRC_URI = \"{ud.url};digest=sha256:{checksum}\""
+        inspect_output = runfetchcmd(f"skopeo inspect docker://{ud.container_name}:{ud.tag}", d, True)
+        digest = json.loads(inspect_output)["Digest"]
 
+        checksum_line = f'SRC_URI = "{ud.url};digest={digest}"'
         strict = d.getVar("BB_STRICT_CHECKSUM") or "0"
 
         # If strict checking enabled and neither sum defined, raise error
         if strict == "1":
             raise NoChecksumError(checksum_line)
 
-        checksum_event = {"sha256sum": checksum}
+        checksum_event = {"sha256sum": digest}
         bb.event.fire(MissingChecksumEvent(ud.url, **checksum_event), d)
 
         if strict == "ignore":
@@ -77,7 +79,7 @@ class Container(FetchMethod):
 
         # Log missing digest so user can more easily add it
         logger.warning(
-            f"Missing checksum for '{ud.localpath}', consider using this " \
+            f"Missing checksum for '{ud.url}', consider using this " \
             f"SRC_URI in the recipe:\n{checksum_line}")
 
     def unpack(self, ud, rootdir, d):
-- 
2.49.0

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20250625135442.1420977-1-clara.kowalsky%40siemens.com.

[PATCH 2/2] container_fetcher: Verify that tag and digest match

['Clara Kowalsky' via isar-users]

If a tag and digest are specified for a container image in the SRC_URI,
the tag is ignored until now and the container image with the matching
digest is fetched.
With this change, the container image is fetched based on the specified
tag and it is checked whether the digest matches. If not, an error is
thrown.

Signed-off-by: Clara Kowalsky <clara.kowalsky@siemens.com>
---
 meta/lib/container_fetcher.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/meta/lib/container_fetcher.py b/meta/lib/container_fetcher.py
index 16467abb..75366988 100644
--- a/meta/lib/container_fetcher.py
+++ b/meta/lib/container_fetcher.py
@@ -11,6 +11,7 @@ from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import logger
 from   bb.fetch2 import MissingChecksumEvent
 from   bb.fetch2 import NoChecksumError
+from   bb.fetch2 import ChecksumError
 from   bb.fetch2 import runfetchcmd
 
 class Container(FetchMethod):
@@ -47,6 +48,22 @@ class Container(FetchMethod):
     def download(self, ud, d):
         tarball = ud.localfile[:-len('.zst')]
         with tempfile.TemporaryDirectory(dir=d.getVar('DL_DIR')) as tmpdir:
+            # If both tag and digest are provided, verify they match
+            if ud.digest and ud.tag != "latest":
+                inspect_output = runfetchcmd(f"skopeo inspect docker://{ud.container_name}:{ud.tag}", d, True)
+                actual_digest = json.loads(inspect_output)["Digest"]
+                if actual_digest != ud.digest:
+                    messages = []
+                    messages.append(f"Checksum mismatch for {ud.container_name}:{ud.tag}")
+                    messages.append("If this change is expected (e.g. you have upgraded " \
+                                "to a new version without updating the checksums) " \
+                                "then you can use these lines within the recipe:")
+                    messages.append(f'SRC_URI = "docker://{ud.container_name};digest={actual_digest};tag={ud.tag}"')
+                    messages.append("Otherwise you should retry the download and/or " \
+                                "check with upstream to determine if the container image has " \
+                                "become corrupted or otherwise unexpectedly modified.")
+                    raise ChecksumError("\n".join(messages), ud.url, actual_digest)
+
             # Take a two steps for downloading into a docker archive because
             # not all source may have the required Docker schema 2 manifest.
             runfetchcmd("skopeo copy --preserve-digests " + \
-- 
2.49.0

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20250625135442.1420977-2-clara.kowalsky%40siemens.com.

Re: [PATCH 2/2] container_fetcher: Verify that tag and digest match

['Jan Kiszka' via isar-users]

On 25.06.25 15:54, Clara Kowalsky wrote:
> If a tag and digest are specified for a container image in the SRC_URI,
> the tag is ignored until now and the container image with the matching
> digest is fetched.
> With this change, the container image is fetched based on the specified
> tag and it is checked whether the digest matches. If not, an error is
> thrown.
> 
> Signed-off-by: Clara Kowalsky <clara.kowalsky@siemens.com>
> ---
>  meta/lib/container_fetcher.py | 17 +++++++++++++++++
>  1 file changed, 17 insertions(+)
> 
> diff --git a/meta/lib/container_fetcher.py b/meta/lib/container_fetcher.py
> index 16467abb..75366988 100644
> --- a/meta/lib/container_fetcher.py
> +++ b/meta/lib/container_fetcher.py
> @@ -11,6 +11,7 @@ from   bb.fetch2 import FetchMethod
>  from   bb.fetch2 import logger
>  from   bb.fetch2 import MissingChecksumEvent
>  from   bb.fetch2 import NoChecksumError
> +from   bb.fetch2 import ChecksumError
>  from   bb.fetch2 import runfetchcmd
>  
>  class Container(FetchMethod):
> @@ -47,6 +48,22 @@ class Container(FetchMethod):
>      def download(self, ud, d):
>          tarball = ud.localfile[:-len('.zst')]
>          with tempfile.TemporaryDirectory(dir=d.getVar('DL_DIR')) as tmpdir:
> +            # If both tag and digest are provided, verify they match
> +            if ud.digest and ud.tag != "latest":

Instead of "latest" (which could have been specified explicitly):

and not "tag" in ud.parm

> +                inspect_output = runfetchcmd(f"skopeo inspect docker://{ud.container_name}:{ud.tag}", d, True)
> +                actual_digest = json.loads(inspect_output)["Digest"]
> +                if actual_digest != ud.digest:
> +                    messages = []
> +                    messages.append(f"Checksum mismatch for {ud.container_name}:{ud.tag}")
> +                    messages.append("If this change is expected (e.g. you have upgraded " \
> +                                "to a new version without updating the checksums) " \
> +                                "then you can use these lines within the recipe:")
> +                    messages.append(f'SRC_URI = "docker://{ud.container_name};digest={actual_digest};tag={ud.tag}"')
> +                    messages.append("Otherwise you should retry the download and/or " \
> +                                "check with upstream to determine if the container image has " \
> +                                "become corrupted or otherwise unexpectedly modified.")

Rather long. Is bitbake similarly verbose when detecting a checksum
mismatch?

> +                    raise ChecksumError("\n".join(messages), ud.url, actual_digest)
> +
>              # Take a two steps for downloading into a docker archive because
>              # not all source may have the required Docker schema 2 manifest.
>              runfetchcmd("skopeo copy --preserve-digests " + \

jan

-- 
Siemens AG, Foundational Technologies
Linux Expert Center

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/6871c387-5857-4cd7-ab93-fbd27d740cc2%40siemens.com.

Re: [PATCH 2/2] container_fetcher: Verify that tag and digest match

['Clara Kowalsky' via isar-users]

On 25.06.25 17:47, Jan Kiszka wrote:
> On 25.06.25 15:54, Clara Kowalsky wrote:
>> If a tag and digest are specified for a container image in the SRC_URI,
>> the tag is ignored until now and the container image with the matching
>> digest is fetched.
>> With this change, the container image is fetched based on the specified
>> tag and it is checked whether the digest matches. If not, an error is
>> thrown.
>>
>> Signed-off-by: Clara Kowalsky <clara.kowalsky@siemens.com>
>> ---
>>   meta/lib/container_fetcher.py | 17 +++++++++++++++++
>>   1 file changed, 17 insertions(+)
>>
>> diff --git a/meta/lib/container_fetcher.py b/meta/lib/container_fetcher.py
>> index 16467abb..75366988 100644
>> --- a/meta/lib/container_fetcher.py
>> +++ b/meta/lib/container_fetcher.py
>> @@ -11,6 +11,7 @@ from   bb.fetch2 import FetchMethod
>>   from   bb.fetch2 import logger
>>   from   bb.fetch2 import MissingChecksumEvent
>>   from   bb.fetch2 import NoChecksumError
>> +from   bb.fetch2 import ChecksumError
>>   from   bb.fetch2 import runfetchcmd
>>   
>>   class Container(FetchMethod):
>> @@ -47,6 +48,22 @@ class Container(FetchMethod):
>>       def download(self, ud, d):
>>           tarball = ud.localfile[:-len('.zst')]
>>           with tempfile.TemporaryDirectory(dir=d.getVar('DL_DIR')) as tmpdir:
>> +            # If both tag and digest are provided, verify they match
>> +            if ud.digest and ud.tag != "latest":
> 
> Instead of "latest" (which could have been specified explicitly):
> 
> and not "tag" in ud.parm

Ok, I'll change that in v2.

> 
>> +                inspect_output = runfetchcmd(f"skopeo inspect docker://{ud.container_name}:{ud.tag}", d, True)
>> +                actual_digest = json.loads(inspect_output)["Digest"]
>> +                if actual_digest != ud.digest:
>> +                    messages = []
>> +                    messages.append(f"Checksum mismatch for {ud.container_name}:{ud.tag}")
>> +                    messages.append("If this change is expected (e.g. you have upgraded " \
>> +                                "to a new version without updating the checksums) " \
>> +                                "then you can use these lines within the recipe:")
>> +                    messages.append(f'SRC_URI = "docker://{ud.container_name};digest={actual_digest};tag={ud.tag}"')
>> +                    messages.append("Otherwise you should retry the download and/or " \
>> +                                "check with upstream to determine if the container image has " \
>> +                                "become corrupted or otherwise unexpectedly modified.")
> 
> Rather long. Is bitbake similarly verbose when detecting a checksum
> mismatch?
> 
Yes, this is actually exactly the same message bitbake is printing in 
case of checksum mismatch: 
https://github.com/ilbers/isar/blob/master/bitbake/lib/bb/fetch2/__init__.py#L633

BR,
Clara

>> +                    raise ChecksumError("\n".join(messages), ud.url, actual_digest)
>> +
>>               # Take a two steps for downloading into a docker archive because
>>               # not all source may have the required Docker schema 2 manifest.
>>               runfetchcmd("skopeo copy --preserve-digests " + \
> 
> jan
> 

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/ec41400d-46ea-4e6f-8570-7b91330efd77%40siemens.com.

Re: [PATCH 1/2] container_fetcher: Fix missing checksum warning

['Jan Kiszka' via isar-users]

On 25.06.25 15:54, Clara Kowalsky wrote:
> In case only a tag is specified for a container image in the SRC_URI and
> no digest, a warning should be issued with the recommendation to add the
> digest of the container image.
> So far, the number specified in the warning would be the checksum of the
> manifest.json, which is a metadata file. However, we want to show the
> registry digest, which is calculated over the complete image content.

Actually, we were presenting the digest of the architecture-specific
image that happened to be fetched first, not that of the manifest
describing images for all supported archs of this tag. I would recommend
to update that.

But the conclusion remains correct: We need the latter, not the former.

Jan

> In addition, reading the manifest.json does not work at this point
> anyway, as skopeo has already packed it into a Docker archive.
> 
> Signed-off-by: Clara Kowalsky <clara.kowalsky@siemens.com>
> ---
>  meta/lib/container_fetcher.py | 10 ++++++----
>  1 file changed, 6 insertions(+), 4 deletions(-)
> 
> diff --git a/meta/lib/container_fetcher.py b/meta/lib/container_fetcher.py
> index 0d659154..16467abb 100644
> --- a/meta/lib/container_fetcher.py
> +++ b/meta/lib/container_fetcher.py
> @@ -6,6 +6,7 @@
>  import oe.path
>  import os
>  import tempfile
> +import json
>  from   bb.fetch2 import FetchMethod
>  from   bb.fetch2 import logger
>  from   bb.fetch2 import MissingChecksumEvent
> @@ -60,16 +61,17 @@ class Container(FetchMethod):
>          if ud.digest:
>              return
>  
> -        checksum = bb.utils.sha256_file(ud.localpath + "/manifest.json")
> -        checksum_line = f"SRC_URI = \"{ud.url};digest=sha256:{checksum}\""
> +        inspect_output = runfetchcmd(f"skopeo inspect docker://{ud.container_name}:{ud.tag}", d, True)
> +        digest = json.loads(inspect_output)["Digest"]
>  
> +        checksum_line = f'SRC_URI = "{ud.url};digest={digest}"'
>          strict = d.getVar("BB_STRICT_CHECKSUM") or "0"
>  
>          # If strict checking enabled and neither sum defined, raise error
>          if strict == "1":
>              raise NoChecksumError(checksum_line)
>  
> -        checksum_event = {"sha256sum": checksum}
> +        checksum_event = {"sha256sum": digest}
>          bb.event.fire(MissingChecksumEvent(ud.url, **checksum_event), d)
>  
>          if strict == "ignore":
> @@ -77,7 +79,7 @@ class Container(FetchMethod):
>  
>          # Log missing digest so user can more easily add it
>          logger.warning(
> -            f"Missing checksum for '{ud.localpath}', consider using this " \
> +            f"Missing checksum for '{ud.url}', consider using this " \
>              f"SRC_URI in the recipe:\n{checksum_line}")
>  
>      def unpack(self, ud, rootdir, d):


-- 
Siemens AG, Foundational Technologies
Linux Expert Center

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/e859f8f5-33ad-417f-a7d5-23a057a7c5dd%40siemens.com.